Pr/1 ontology driven core

.gitignore

@@ -136,13 +136,16 @@ dmypy.json
 
 #Jims Development
 stata/various_stata_development__files
+playing/
 
 # outputs folder
 /notebooks/outputs/
+/notebooks/old/
 
 # test output
 /test.xlsx
 /outputs/
 
-# test datasets that can't b shared
+# test datasets that can't be shared
 UKDA-7401-stata12/
+notebooks/old/test.ipynb

acro/acro.py

@@ -11,7 +11,7 @@
 
 import yaml
 
-from . import acro_tables
+from . import acro_tables, sdcchecks
 from .acro_regression import Regression
 from .acro_tables import Tables
 from .record import Records
@@ -59,6 +59,7 @@ def __init__(
         suppress: bool = False,
         mitigation: str | None = None,
         round_base: int | None = None,
+        federated: bool | None = None,
     ) -> None:
         """Construct a new ACRO object and reads parameters from config.
 
@@ -76,36 +77,36 @@ def __init__(
         round_base : int, optional
             The base to round to when ``mitigation="round"``. Defaults to the
             ``safe_round_base`` value from the yaml config.
+        federated : bool, optional
+            Whether to run in federated mode. When ``True``, checks are skipped
+            and evidence is written to ``evidence.json`` for a trusted
+            aggregator. When ``None``, falls back to the yaml config value
+            (default ``False``).
         """
         Tables.__init__(self, suppress=suppress, mitigation=mitigation)
         Regression.__init__(self, config)
+        self._federated_override = federated  # applied after yaml is loaded
         self.config: dict[str, Any] = {}
         path: pathlib.Path = pathlib.Path(__file__).with_name(config + ".yaml")
         logger.debug("path: %s", path)
         with open(path, encoding="utf-8") as handle:
             self.config = yaml.load(handle, Loader=yaml.loader.SafeLoader)
+
         # Tables and Regression each construct their own ``self.results`` so
         # they can be used standalone; in the combined ACRO object we want a
         # single shared Records instance regardless of init order, so make
         # that explicit here.
         self.results: Records = Records(
             blocked_extensions=self.config.get("blocked_extensions", [])
         )
+
         # Preserve the user-requested suppress value alongside the yaml config.
         # Tables.suppress is now a property derived from the live mitigation, so
         # it can drift over a session (enable_rounding flips mitigation away from
         # 'suppress'). Recording the original ask here keeps it available for
         # callers that need to re-enforce suppression for disclosive outputs.
         self.config["suppress"] = suppress
-        # set globals needed for aggregation functions
-        acro_tables.THRESHOLD = self.config["safe_threshold"]
-        acro_tables.SAFE_PRATIO_P = self.config["safe_pratio_p"]
-        acro_tables.SAFE_NK_N = self.config["safe_nk_n"]
-        acro_tables.SAFE_NK_K = self.config["safe_nk_k"]
-        acro_tables.CHECK_MISSING_VALUES = self.config["check_missing_values"]
-        acro_tables.ZEROS_ARE_DISCLOSIVE = self.config["zeros_are_disclosive"]
-        # set globals for survival analysis
-        acro_tables.SURVIVAL_THRESHOLD = self.config["survival_safe_threshold"]
+
         # set globals and instance state for the round mitigation strategy
         acro_tables.SAFE_ROUND_BASE = self.config.get(
             "safe_round_base", acro_tables.SAFE_ROUND_BASE
@@ -117,6 +118,16 @@ def __init__(
         logger.info("config: %s", self.config)
         logger.info("mitigation: %s (round_base=%s)", self.mitigation, self.round_base)
 
+        # make object to handle all the ontology-driven checking
+        self.sdc_checks = sdcchecks.SDCChecks(self.config)
+
+        if self._federated_override is not None:
+            self.federated: bool = bool(self._federated_override)
+        else:
+            self.federated = bool(self.config.get("federated", False))
+        self.config["federated"] = self.federated
+        logger.info("federated: %s", self.federated)
+
     def finalise(
         self, path: str = "outputs", ext: str = "json", interactive: bool = False
     ) -> Records | None:
@@ -140,11 +151,23 @@ def finalise(
         if os.path.exists(path):
             logger.warning(
                 "Results file can not be created. "
-                "Directory %s already exists. Please choose a different directory name.",
+                "Directory %s already exists. "
+                "Please choose a different directory name.",
                 path,
             )
             return None
-        self.results.finalise(path, ext, interactive)
+
+        if self.federated:
+            os.makedirs(path, exist_ok=True)
+            merged_evidence: dict = dict(getattr(self, "_federated_evidence", {}))
+            evidence_data = self.results.finalise_evidence(path, merged_evidence)
+            evidence_filename: str = os.path.normpath(f"{path}/evidence.json")
+            with open(evidence_filename, "w", newline="", encoding="utf-8") as fh:
+                json.dump(evidence_data, fh, indent=4, sort_keys=False)
+            logger.info("federated evidence written to: %s", path)
+        else:
+            self.results.finalise(path, ext, interactive)
+
         config_filename: str = os.path.normpath(f"{path}/config.json")
         try:
             with open(config_filename, "w", newline="", encoding="utf-8") as file:
@@ -238,16 +261,29 @@ def disable_suppression(self) -> None:
         self.suppress = False
 
     def enable_rounding(self, base: int | None = None) -> None:
-        """Turn rounding on; overwrites any prior suppress=True (not restored on disable_rounding)."""
+        """Turn rounding on. Overwrites any prior suppress=True (not restored on disable_rounding)."""
         if base is not None:
             self.round_base = base
         self.mitigation = "round"
 
     def disable_rounding(self) -> None:
-        """Turn rounding off; always falls back to mitigation='none' (prior suppress not restored)."""
+        """Turn rounding off. Always falls back to mitigation='none' (prior suppress not restored)."""
         if self.mitigation == "round":
             self.mitigation = "none"
 
+    def show_fair_summaries(self) -> str:
+        """Print ids and fair summaries for all outputs in session."""
+        thestr = ""
+        for uid, value in self.results.results.items():
+            thestr += uid + "\n"
+            for key, val in value.fair.items():
+                if isinstance(val, dict):
+                    for key2, val2 in val.items():
+                        thestr += f"{key2} : {val2}"
+                else:
+                    thestr += f"{key}:{val}"
+        return thestr + "\n"
+
 
 def add_to_acro(src_path: str, dest_path: str = "sdc_results") -> None:
     """Add outputs to an acro object and creates a results file for checking.
@@ -260,11 +296,9 @@ def add_to_acro(src_path: str, dest_path: str = "sdc_results") -> None:
         Name of the folder to save outputs.
     """
     acro: ACRO = ACRO()
-    output_id: int = 0
     # add the files from the folder to an acro obj
-    for file in os.listdir(src_path):
+    for output_id, file in enumerate(os.listdir(src_path)):
         filename: str = os.path.join(src_path, file)
         acro.custom_output(filename)
         acro.rename_output(f"output_{output_id}", file)
-        output_id += 1
     acro.finalise(dest_path, "json")