HDXDSYS-1822 Check issue with HDX Analysis Scripts output raised by Melanie

.github/workflows/run-python-script-datasetsinfo.yml

@@ -5,7 +5,7 @@ name: run_datasetsinfo
 on:
   workflow_dispatch: # add run button in github
   schedule:
-    - cron: "10 0 1 */3 *"
+    - cron: "43 4 * * *"
 
 jobs:
   run:

common/dataset_statistics.py

@@ -174,6 +174,7 @@ def get_updated_by_script(self):
         self.updated_by_noncod_script = "N"
         self.updated_by_cod_script = "N"
         self.old_updated_by_noncod_script = "N"
+        self.old_updated_by_cod_script = "N"
         self.outdated_lastmodified = "N"
         if not updated_by_script:
             return
@@ -204,9 +205,12 @@ def get_updated_by_script(self):
         if (
             "HDXINTERNAL" in updated_by_script
             and "CODs" in updated_by_script
-            and "cod_level" in self.data
         ):
-            self.updated_by_cod_script = "Y"
+            if "cod_level" in self.data:
+                self.updated_by_cod_script = "Y"
+            else:
+                # no longer updated by COD script
+                self.old_updated_by_cod_script = "Y"
             return
 
         if self.last_modified:

get_datasets_info.py

@@ -59,6 +59,7 @@ def main(downloads, output_dir, **ignore):
             "requestable",
             "archived",
             "updated by cod script",
+            "formerly updated by cod script",
             "updated by non-cod script",
             "date updated by script",
             "updated_by_script<<last_modified",
@@ -125,6 +126,7 @@ def main(downloads, output_dir, **ignore):
             datasetstats.requestable,
             datasetstats.archived,
             datasetstats.updated_by_cod_script,
+            datasetstats.old_updated_by_cod_script,
             datasetstats.updated_by_noncod_script,
             datasetstats.updated_by_script,
             datasetstats.old_updated_by_noncod_script,

get_org_stats.py

@@ -93,6 +93,7 @@ def main(downloads, output_dir, **ignore):
         organisation["public internal resources"] = 0
         organisation["public external resources"] = 0
         organisation["updated by cod script"] = 0
+        organisation["formerly updated by cod script"] = 0
         organisation["updated by script"] = 0
         organisation["old updated by script"] = 0
         organisation["any updated last 3 months"] = "No"
@@ -212,6 +213,11 @@ def main(downloads, output_dir, **ignore):
         ):
             organisation["updated by cod script"] += 1
             total_updated_by_cod += 1
+        if (
+            datasetstats.old_updated_by_cod_script == "Y"
+            and is_public_not_requestable_archived
+        ):
+            organisation["formerly updated by cod script"] += 1
         if datasetstats.created > organisation["latest created dataset date"]:
             organisation["latest created dataset date"] = datasetstats.created
         if datasetstats.updated_by_script:
@@ -256,6 +262,8 @@ def main(downloads, output_dir, **ignore):
         "% of public API (non-cod scripted)",
         "Public cod scripted",
         "% of public cod scripted",
+        "Public formerly cod scripted",
+        "% of public formerly cod scripted",
         "Public previous scripted",
         "% of public previous scripted",
         "Public live",
@@ -305,6 +313,9 @@ def get_number_percentage(organisation, key):
         updated_by_cod_script, percentage_cod = get_number_percentage(
             organisation, "updated by cod script"
         )
+        old_updated_by_cod_script, old_percentage_cod = get_number_percentage(
+            organisation, "formerly updated by cod script"
+        )
         updated_by_api, percentage_api = get_number_percentage(
             organisation, "updated by script"
         )
@@ -353,6 +364,8 @@ def get_number_percentage(organisation, key):
             percentage_api,
             updated_by_cod_script,
             percentage_cod,
+            old_updated_by_cod_script,
+            old_percentage_cod,
             old_updated_by_script,
             percentage_old_script,
             live_datasets,

tests/fixtures/total_stats.csv

@@ -1,2 +1,2 @@
 Public - Request & Archive,Public Internal Resources,Public External Resources,Updated by COD,Updated by Script,Quarterly % API OKR,Last Modified Fresh,Last Modified Not Fresh,Quarterly % Last Modified Fresh OKR,End Date Up to Date,End Date Out Of Date,Quarterly % End Date Up To Date OKR
-20823,54050,110729,583,16594,80,12685,8135,61,5955,9987,37
+20823,54050,110729,583,16593,80,12685,8135,61,5955,9987,37

tests/test_get_org_stats.py

@@ -15,7 +15,7 @@ def test_get_org_stats(self, configuration, fixtures, mock_downloads):
             )
             assert total_public == 20823
             assert total_updated_by_cod == 583
-            assert total_updated_by_script == 16594
+            assert total_updated_by_script == 16593
             filename = "org_stats.csv"
             assert_files_same(join(fixtures, filename), join(folder, filename))
             filename = "total_stats.csv"