From 80f91555dab1429be511b6f348664712ab7ff39e Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Wed, 24 Jun 2026 10:49:38 +0100 Subject: [PATCH 1/6] update_demo_notebook --- notebooks/acro_demo_2026.ipynb | 346 ++++++++++++++++++--------------- 1 file changed, 190 insertions(+), 156 deletions(-) diff --git a/notebooks/acro_demo_2026.ipynb b/notebooks/acro_demo_2026.ipynb index a668d98d..c26d6b42 100644 --- a/notebooks/acro_demo_2026.ipynb +++ b/notebooks/acro_demo_2026.ipynb @@ -315,15 +315,19 @@ "\n", "The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for\n", "\n", - "### Example 1: A simple 2-D table of frequencies stratified by two variables\n", - "\n", + "### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categoriacl variables\n", "Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write\n", "````\n", "pd.crosstab(...)\n", "````\n", "so the only change is to use the prefix `acro.` rather than `pd.`\n", "\n", - "_NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing.\n", + "_NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing.
\n", + "\n", + "Going through the parameters in order:\n", + "- `index=df.recommendation` and `columns=df.parents` tell it we want to create a cross tabulation by two categorical variables: recommendation and parents \n", + "- setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell)\n", + "- setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*)\n", "\n", "Now run the next cell." ] @@ -337,18 +341,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:acro:get_summary(): fail; threshold: 4 cells may need suppressing; \n", + "INFO:acro:get_summary(): fail; threshold: 1 cells may need suppressing; p-ratio: 4 cells may need suppressing; nk-rule: 4 cells may need suppressing; \n", "INFO:acro:outcome_df:\n", - "--------------------------------------------------------|\n", - "parents |great_pret |pretentious |usual |\n", - "recommendation | | | |\n", - "--------------------------------------------------------|\n", - "not_recom | ok | ok | ok|\n", - "priority | ok | ok | ok|\n", - "recommend | threshold; | threshold; | threshold; |\n", - "spec_prior | ok | ok | ok|\n", - "very_recom | threshold; | ok | ok|\n", - "--------------------------------------------------------|\n", + "----------------------------------------------------------------------------------------|\n", + "parents |great_pret |pretentious |usual |\n", + "recommendation | | | |\n", + "----------------------------------------------------------------------------------------|\n", + "not_recom | ok | ok | ok|\n", + "priority | ok | ok | ok|\n", + "recommend | p-ratio; nk-rule; | p-ratio; nk-rule; | threshold; p-ratio; nk-rule; |\n", + "spec_prior | ok | ok | ok|\n", + "very_recom | p-ratio; nk-rule; | ok | ok|\n", + "----------------------------------------------------------------------------------------|\n", "\n", "INFO:acro:records:add(): output_0\n" ] @@ -388,46 +392,46 @@ " \n", " \n", " not_recom\n", - " 1440\n", - " 1440\n", - " 1440\n", + " 3.125000\n", + " 3.114583\n", + " 3.086806\n", " \n", " \n", " priority\n", - " 858\n", - " 1484\n", - " 1924\n", + " 2.578089\n", + " 3.056604\n", + " 3.153846\n", " \n", " \n", " recommend\n", - " 0\n", - " 0\n", - " 2\n", + " NaN\n", + " NaN\n", + " 1.000000\n", " \n", " \n", " spec_prior\n", - " 2022\n", - " 1264\n", - " 758\n", + " 3.386746\n", + " 3.318829\n", + " 3.358839\n", " \n", " \n", " very_recom\n", - " 0\n", - " 132\n", - " 196\n", + " NaN\n", + " 2.212121\n", + " 2.198980\n", " \n", " \n", "\n", "" ], "text/plain": [ - "parents great_pret pretentious usual\n", - "recommendation \n", - "not_recom 1440 1440 1440\n", - "priority 858 1484 1924\n", - "recommend 0 0 2\n", - "spec_prior 2022 1264 758\n", - "very_recom 0 132 196" + "parents great_pret pretentious usual\n", + "recommendation \n", + "not_recom 3.125000 3.114583 3.086806\n", + "priority 2.578089 3.056604 3.153846\n", + "recommend NaN NaN 1.000000\n", + "spec_prior 3.386746 3.318829 3.358839\n", + "very_recom NaN 2.212121 2.198980" ] }, "execution_count": 5, @@ -436,7 +440,9 @@ } ], "source": [ - "acro.crosstab(index=df.recommendation, columns=df.parents)" + "acro.crosstab(\n", + " index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\"\n", + ")" ] }, { @@ -446,13 +452,23 @@ "### How to understand this output\n", "The top part (with a pink background) is the risk analysis produced by acro. \n", "It is telling us that:\n", - "- the overall summary is _fail_ because 4 cells are failing the 'minimum threshold' check\n", + "- the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check\n", "- then it is showing which cells failed so you can choose how to respond\n", "- finally it is telling us that is has saved the table and risk assessment to our acro session with id \"output_0\"\n", "\n", - "The part below is the normal output produced by the pandas _crosstab()_ function. \n", - "- As this is such a small table it is not hard to spot the four problematic cells with zero or low counts\n", - "- but of course this might be harder for a bigger table.\n", + "The part below is the normal output produced by the pandas _crosstab()_ function.\n", + "\n", + "- In this table, it might be easy to identifying \"Obvious\" Risks. It is simple to spot that empty cells (where values are NaN) can be disclosive. If the mean is NaN, it means no records in the dataset match those specific criteria. For example, a NaN in the first column and third row indicates that there are no children whose parents are great_pret and whose recommendation is recommend. This information itself can be disclosive.\n", + "\n", + "- But it is much harder to spot the \"Hidden\" Risks (for example a cell with a mean of 10) just by looking at this table. You always need more information than what is provided in a single table. \n", + "\n", + "-In this table, you would need:\n", + " - A secondary count table to verify if the number of records in that cell falls below a required threshold. \n", + " - To assess the risk of dominance (calculate the p-ratio), you would need to extract all individual records that fall into that specific cell from the original dataset, sort them, and manually calculate the ratio to check it against the TRE (Trusted Research Environment) appetite.\n", + " - Another check for dominance is The NK-rule which is even more complicated.\n", + " \n", + " Imagine doing this for much larger tables! Actually acro can do this for us in the background!\n", + "\n", "\n", "### How to respond to this input\n", "There are basically three choices:\n", @@ -470,7 +486,7 @@ "3. We can redact the disclosive cells - and **acro will do this for us**.
\n", "We simply enable the option to suppress disclosive cells and re-run the query.\n", "\n", - "The cell below shows option 3.\n", + "The cell below shows option 3.b\n", "When you run the cell below you should see that:\n", "- the status now changes to `review` (so the output-checker knows what has been applied)\n", "- the code automatically adds an exception request saying that suppression has been applied\n", @@ -486,18 +502,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:acro:get_summary(): review; threshold: 4 cells suppressed; \n", + "INFO:acro:get_summary(): review; threshold: 1 cells suppressed; p-ratio: 4 cells suppressed; nk-rule: 4 cells suppressed; \n", "INFO:acro:outcome_df:\n", - "--------------------------------------------------------|\n", - "parents |great_pret |pretentious |usual |\n", - "recommendation | | | |\n", - "--------------------------------------------------------|\n", - "not_recom | ok | ok | ok|\n", - "priority | ok | ok | ok|\n", - "recommend | threshold; | threshold; | threshold; |\n", - "spec_prior | ok | ok | ok|\n", - "very_recom | threshold; | ok | ok|\n", - "--------------------------------------------------------|\n", + "----------------------------------------------------------------------------------------|\n", + "parents |great_pret |pretentious |usual |\n", + "recommendation | | | |\n", + "----------------------------------------------------------------------------------------|\n", + "not_recom | ok | ok | ok|\n", + "priority | ok | ok | ok|\n", + "recommend | p-ratio; nk-rule; | p-ratio; nk-rule; | threshold; p-ratio; nk-rule; |\n", + "spec_prior | ok | ok | ok|\n", + "very_recom | p-ratio; nk-rule; | ok | ok|\n", + "----------------------------------------------------------------------------------------|\n", "\n", "INFO:acro:records:add(): output_1\n", "INFO:acro:records:exception request was added to output_1\n" @@ -538,15 +554,15 @@ " \n", " \n", " not_recom\n", - " 1440.0\n", - " 1440.0\n", - " 1440.0\n", + " 3.125000\n", + " 3.114583\n", + " 3.086806\n", " \n", " \n", " priority\n", - " 858.0\n", - " 1484.0\n", - " 1924.0\n", + " 2.578089\n", + " 3.056604\n", + " 3.153846\n", " \n", " \n", " recommend\n", @@ -556,28 +572,28 @@ " \n", " \n", " spec_prior\n", - " 2022.0\n", - " 1264.0\n", - " 758.0\n", + " 3.386746\n", + " 3.318829\n", + " 3.358839\n", " \n", " \n", " very_recom\n", " NaN\n", - " 132.0\n", - " 196.0\n", + " 2.212121\n", + " 2.198980\n", " \n", " \n", "\n", "" ], "text/plain": [ - "parents great_pret pretentious usual\n", - "recommendation \n", - "not_recom 1440.0 1440.0 1440.0\n", - "priority 858.0 1484.0 1924.0\n", - "recommend NaN NaN NaN\n", - "spec_prior 2022.0 1264.0 758.0\n", - "very_recom NaN 132.0 196.0" + "parents great_pret pretentious usual\n", + "recommendation \n", + "not_recom 3.125000 3.114583 3.086806\n", + "priority 2.578089 3.056604 3.153846\n", + "recommend NaN NaN NaN\n", + "spec_prior 3.386746 3.318829 3.358839\n", + "very_recom NaN 2.212121 2.198980" ] }, "execution_count": 6, @@ -587,7 +603,9 @@ ], "source": [ "acro.enable_suppression()\n", - "acro.crosstab(index=df.recommendation, columns=df.parents)" + "acro.crosstab(\n", + " index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\"\n", + ")" ] }, { @@ -677,62 +695,62 @@ " \n", " great_pret\n", " convenient\n", - " 3.104167\n", - " 2.789062\n", - " 3.320043\n", + " 3.111111\n", + " 2.753906\n", + " 3.354526\n", " NaN\n", - " 3.122222\n", + " 3.131019\n", " \n", " \n", " inconv\n", - " 3.123611\n", - " 2.401734\n", - " 3.372943\n", + " 3.138889\n", + " 2.317919\n", + " 3.414077\n", " NaN\n", - " 3.134259\n", + " 3.146759\n", " \n", " \n", " pretentious\n", " convenient\n", - " 3.065278\n", - " 3.058594\n", - " 3.289384\n", - " 2.590909\n", - " 3.104167\n", + " 3.123611\n", + " 3.072917\n", + " 3.304795\n", + " 2.636364\n", + " 3.134722\n", " \n", " \n", " inconv\n", - " 3.008333\n", - " 2.997207\n", - " 3.345588\n", + " 3.105556\n", + " 3.039106\n", + " 3.330882\n", " 1.363636\n", - " 3.077315\n", + " 3.118981\n", " \n", " \n", " usual\n", " convenient\n", - " 3.134722\n", - " 3.135892\n", - " 3.325581\n", - " 2.607692\n", - " 3.133920\n", + " 3.050000\n", + " 3.117220\n", + " 3.293605\n", + " 2.623077\n", + " 3.093142\n", " \n", " \n", " inconv\n", - " 3.102778\n", - " 3.075000\n", - " 3.362319\n", + " 3.123611\n", + " 3.190625\n", + " 3.413043\n", " 1.363636\n", - " 3.087037\n", + " 3.155093\n", " \n", " \n", " All\n", " \n", - " 3.089815\n", - " 2.983826\n", - " 3.339021\n", - " 2.185976\n", - " 3.109816\n", + " 3.108796\n", + " 3.004219\n", + " 3.360287\n", + " 2.204268\n", + " 3.129958\n", " \n", " \n", "\n", @@ -741,13 +759,13 @@ "text/plain": [ "recommendation not_recom priority spec_prior very_recom All\n", "parents finance \n", - "great_pret convenient 3.104167 2.789062 3.320043 NaN 3.122222\n", - " inconv 3.123611 2.401734 3.372943 NaN 3.134259\n", - "pretentious convenient 3.065278 3.058594 3.289384 2.590909 3.104167\n", - " inconv 3.008333 2.997207 3.345588 1.363636 3.077315\n", - "usual convenient 3.134722 3.135892 3.325581 2.607692 3.133920\n", - " inconv 3.102778 3.075000 3.362319 1.363636 3.087037\n", - "All 3.089815 2.983826 3.339021 2.185976 3.109816" + "great_pret convenient 3.111111 2.753906 3.354526 NaN 3.131019\n", + " inconv 3.138889 2.317919 3.414077 NaN 3.146759\n", + "pretentious convenient 3.123611 3.072917 3.304795 2.636364 3.134722\n", + " inconv 3.105556 3.039106 3.330882 1.363636 3.118981\n", + "usual convenient 3.050000 3.117220 3.293605 2.623077 3.093142\n", + " inconv 3.123611 3.190625 3.413043 1.363636 3.155093\n", + "All 3.108796 3.004219 3.360287 2.204268 3.129958" ] }, "execution_count": 7, @@ -863,24 +881,32 @@ "status: fail\n", "type: table\n", "properties: {'method': 'crosstab'}\n", - "sdc: {'summary': {'suppressed': False, 'negative': 0, 'missing': 0, 'threshold': 4, 'p-ratio': 0, 'nk-rule': 0, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 0], [2, 1], [2, 2], [4, 0]], 'p-ratio': [], 'nk-rule': [], 'all-values-are-same': []}}\n", - "command: acro.crosstab(index=df.recommendation, columns=df.parents)\n", - "summary: fail; threshold: 4 cells may need suppressing; \n", - "outcome: parents great_pret pretentious usual\n", - "recommendation \n", - "not_recom ok ok ok\n", - "priority ok ok ok\n", - "recommend threshold; threshold; threshold; \n", - "spec_prior ok ok ok\n", - "very_recom threshold; ok ok\n", - "output: [parents great_pret pretentious usual\n", - "recommendation \n", - "not_recom 1440 1440 1440\n", - "priority 858 1484 1924\n", - "recommend 0 0 2\n", - "spec_prior 2022 1264 758\n", - "very_recom 0 132 196]\n", - "timestamp: 2026-02-11T18:33:37.547019\n", + "sdc: {'summary': {'suppressed': False, 'negative': 0, 'missing': 0, 'threshold': 1, 'p-ratio': 4, 'nk-rule': 4, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 2]], 'p-ratio': [[2, 0], [2, 1], [2, 2], [4, 0]], 'nk-rule': [[2, 0], [2, 1], [2, 2], [4, 0]], 'all-values-are-same': []}}\n", + "command: acro.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")\n", + "summary: fail; threshold: 1 cells may need suppressing; p-ratio: 4 cells may need suppressing; nk-rule: 4 cells may need suppressing; \n", + "outcome: parents great_pret pretentious \\\n", + "recommendation \n", + "not_recom ok ok \n", + "priority ok ok \n", + "recommend p-ratio; nk-rule; p-ratio; nk-rule; \n", + "spec_prior ok ok \n", + "very_recom p-ratio; nk-rule; ok \n", + "\n", + "parents usual \n", + "recommendation \n", + "not_recom ok \n", + "priority ok \n", + "recommend threshold; p-ratio; nk-rule; \n", + "spec_prior ok \n", + "very_recom ok \n", + "output: [parents great_pret pretentious usual\n", + "recommendation \n", + "not_recom 3.125000 3.114583 3.086806\n", + "priority 2.578089 3.056604 3.153846\n", + "recommend NaN NaN 1.000000\n", + "spec_prior 3.386746 3.318829 3.358839\n", + "very_recom NaN 2.212121 2.198980]\n", + "timestamp: 2026-06-24T10:38:39.678463\n", "comments: []\n", "exception: \n", "\n", @@ -888,24 +914,32 @@ "status: review\n", "type: table\n", "properties: {'method': 'crosstab'}\n", - "sdc: {'summary': {'suppressed': True, 'negative': 0, 'missing': 0, 'threshold': 4, 'p-ratio': 0, 'nk-rule': 0, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 0], [2, 1], [2, 2], [4, 0]], 'p-ratio': [], 'nk-rule': [], 'all-values-are-same': []}}\n", - "command: acro.crosstab(index=df.recommendation, columns=df.parents)\n", - "summary: review; threshold: 4 cells suppressed; \n", - "outcome: parents great_pret pretentious usual\n", - "recommendation \n", - "not_recom ok ok ok\n", - "priority ok ok ok\n", - "recommend threshold; threshold; threshold; \n", - "spec_prior ok ok ok\n", - "very_recom threshold; ok ok\n", - "output: [parents great_pret pretentious usual\n", + "sdc: {'summary': {'suppressed': True, 'negative': 0, 'missing': 0, 'threshold': 1, 'p-ratio': 4, 'nk-rule': 4, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 2]], 'p-ratio': [[2, 0], [2, 1], [2, 2], [4, 0]], 'nk-rule': [[2, 0], [2, 1], [2, 2], [4, 0]], 'all-values-are-same': []}}\n", + "command: acro.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")\n", + "summary: review; threshold: 1 cells suppressed; p-ratio: 4 cells suppressed; nk-rule: 4 cells suppressed; \n", + "outcome: parents great_pret pretentious \\\n", + "recommendation \n", + "not_recom ok ok \n", + "priority ok ok \n", + "recommend p-ratio; nk-rule; p-ratio; nk-rule; \n", + "spec_prior ok ok \n", + "very_recom p-ratio; nk-rule; ok \n", + "\n", + "parents usual \n", "recommendation \n", - "not_recom 1440.0 1440.0 1440.0\n", - "priority 858.0 1484.0 1924.0\n", - "recommend NaN NaN NaN\n", - "spec_prior 2022.0 1264.0 758.0\n", - "very_recom NaN 132.0 196.0]\n", - "timestamp: 2026-02-11T18:33:37.566599\n", + "not_recom ok \n", + "priority ok \n", + "recommend threshold; p-ratio; nk-rule; \n", + "spec_prior ok \n", + "very_recom ok \n", + "output: [parents great_pret pretentious usual\n", + "recommendation \n", + "not_recom 3.125000 3.114583 3.086806\n", + "priority 2.578089 3.056604 3.153846\n", + "recommend NaN NaN NaN\n", + "spec_prior 3.386746 3.318829 3.358839\n", + "very_recom NaN 2.212121 2.198980]\n", + "timestamp: 2026-06-24T10:38:39.772559\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -937,14 +971,14 @@ "All ok ok ok \n", "output: [recommendation not_recom priority spec_prior very_recom All\n", "parents finance \n", - "great_pret convenient 3.104167 2.789062 3.320043 NaN 3.122222\n", - " inconv 3.123611 2.401734 3.372943 NaN 3.134259\n", - "pretentious convenient 3.065278 3.058594 3.289384 2.590909 3.104167\n", - " inconv 3.008333 2.997207 3.345588 1.363636 3.077315\n", - "usual convenient 3.134722 3.135892 3.325581 2.607692 3.133920\n", - " inconv 3.102778 3.075000 3.362319 1.363636 3.087037\n", - "All 3.089815 2.983826 3.339021 2.185976 3.109816]\n", - "timestamp: 2026-02-11T18:33:37.754124\n", + "great_pret convenient 3.111111 2.753906 3.354526 NaN 3.131019\n", + " inconv 3.138889 2.317919 3.414077 NaN 3.146759\n", + "pretentious convenient 3.123611 3.072917 3.304795 2.636364 3.134722\n", + " inconv 3.105556 3.039106 3.330882 1.363636 3.118981\n", + "usual convenient 3.050000 3.117220 3.293605 2.623077 3.093142\n", + " inconv 3.123611 3.190625 3.413043 1.363636 3.155093\n", + "All 3.108796 3.004219 3.360287 2.204268 3.129958]\n", + "timestamp: 2026-06-24T10:38:40.198688\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -1140,9 +1174,9 @@ ], "metadata": { "kernelspec": { - "display_name": "sacro2025", + "display_name": "venv", "language": "python", - "name": "sacro2025" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -1154,7 +1188,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.11" + "version": "3.13.0" } }, "nbformat": 4, From 8f38ca2525977591af7c1d0f53fd5800068ba72e Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Wed, 24 Jun 2026 13:08:46 +0100 Subject: [PATCH 2/6] updating acro_demo_2026.py --- notebooks/acro_demo_2026.ipynb | 2 +- notebooks/acro_demo_2026.py | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/notebooks/acro_demo_2026.ipynb b/notebooks/acro_demo_2026.ipynb index c26d6b42..06cec9b9 100644 --- a/notebooks/acro_demo_2026.ipynb +++ b/notebooks/acro_demo_2026.ipynb @@ -315,7 +315,7 @@ "\n", "The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for\n", "\n", - "### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categoriacl variables\n", + "### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categorical variables\n", "Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write\n", "````\n", "pd.crosstab(...)\n", diff --git a/notebooks/acro_demo_2026.py b/notebooks/acro_demo_2026.py index 9b2bb0ae..a3df73ed 100644 --- a/notebooks/acro_demo_2026.py +++ b/notebooks/acro_demo_2026.py @@ -150,7 +150,7 @@ # # The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for # -# ### Example 1: A simple 2-D table of frequencies stratified by two variables +# ### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categorical variables # # Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write # ```` @@ -160,24 +160,39 @@ # # _NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing. # +# Going through the parameters in order: +# - `index=df.recommendation` and `columns=df.parents` tell it we want to create a cross tabulation by two categorical variables: recommendation and parents +# - setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell) +# - setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*) +# # Now run the next cell. # In[5]: -acro.crosstab(index=df.recommendation, columns=df.parents) +acro.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) # ### How to understand this output # The top part (with a pink background) is the risk analysis produced by acro. # It is telling us that: -# - the overall summary is _fail_ because 4 cells are failing the 'minimum threshold' check +# - the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check # - then it is showing which cells failed so you can choose how to respond # - finally it is telling us that is has saved the table and risk assessment to our acro session with id "output_0" # # The part below is the normal output produced by the pandas _crosstab()_ function. -# - As this is such a small table it is not hard to spot the four problematic cells with zero or low counts -# - but of course this might be harder for a bigger table. +# - In this table, it might be easy to identifying "Obvious" Risks. It is simple to spot that empty cells (where values are NaN) can be disclosive. If the mean is NaN, it means no records in the dataset match those specific criteria. For example, a NaN in the first column and third row indicates that there are no children whose parents are great_pret and whose recommendation is recommend. This information itself can be disclosive. + +# - But it is much harder to spot the "Hidden" Risks (for example a cell with a mean of 10) just by looking at this table. You always need more information than what is provided in a single table. + +# -In this table, you would need: +# - A secondary count table to verify if the number of records in that cell falls below a required threshold. +# - To assess the risk of dominance (calculate the p-ratio), you would need to extract all individual records that fall into that specific cell from the original dataset, sort them, and manually calculate the ratio to check it against the TRE (Trusted Research Environment) appetite. +# - Another check for dominance is The NK-rule which is even more complicated. +# +# Imagine doing this for much larger tables! Actually acro can do this for us in the background! # # ### How to respond to this input # There are basically three choices: @@ -205,8 +220,9 @@ acro.enable_suppression() -acro.crosstab(index=df.recommendation, columns=df.parents) - +acro.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) # ## An example of a more complex table # Just to show off the sort of tables that `cross_tab()` can produce, let's make something more complex.
From b14ad3568720f68624648b6ff52cb56fcc9c722d Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Tue, 30 Jun 2026 15:17:52 +0100 Subject: [PATCH 3/6] updating notebooks --- notebooks/acro_demo_2026.ipynb | 454 ++++++++++++++++++++------------- notebooks/acro_demo_2026.py | 281 ++++++++++---------- 2 files changed, 421 insertions(+), 314 deletions(-) diff --git a/notebooks/acro_demo_2026.ipynb b/notebooks/acro_demo_2026.ipynb index 06cec9b9..909a8c75 100644 --- a/notebooks/acro_demo_2026.ipynb +++ b/notebooks/acro_demo_2026.ipynb @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 71, "metadata": {}, "outputs": [ { @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -289,7 +289,7 @@ "4 priority priority " ] }, - "execution_count": 4, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -306,6 +306,8 @@ "source": [ "## C: Producing tables that are 'Safe Outputs'\n", "\n", + "### Producing tables using standard python libraries\n", + "\n", "The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. \n", "- There are hundreds (thousands?) of web sites showing how to do this.\n", "- You can make (hierarchical) 2-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row)\n", @@ -313,28 +315,147 @@ " - providing a statistic - for example: mean, count, std deviation, median etc.(pandas calls these *aggregation functions*)\n", " - specifying what variable to report on\n", "\n", - "The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for\n", + "In the next example:\n", + "- We will use the Pandas *crosstab()* function to create a summary table.\n", + "\n", + "- The table will calculate the **mean number of children** for every combination of **Recommendation** and **Parental status**. Going through the parameters in order:\n", + " - `index=df.recommendation`: Defines the rows of the table using the recommendation column from the dataset.\n", + "\n", + " - `columns=df.parents`: Defines the columns of the table using the parents column from the dataset.\n", + " - `values=df.children`: Specifies that we want to perform calculations on the children (the name of a column in the dataset).\n", + "\n", + " - `aggfunc=\"mean\"`: Tells the function to calculate the **average (mean)** for those values." + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
parentsgreat_pretpretentioususual
recommendation
not_recom3.1062503.1159723.143056
priority2.6212123.0168463.108628
recommendNaNNaN1.000000
spec_prior3.3209693.3267413.357520
very_recomNaN2.1969702.239796
\n", + "
" + ], + "text/plain": [ + "parents great_pret pretentious usual\n", + "recommendation \n", + "not_recom 3.106250 3.115972 3.143056\n", + "priority 2.621212 3.016846 3.108628\n", + "recommend NaN NaN 1.000000\n", + "spec_prior 3.320969 3.326741 3.357520\n", + "very_recom NaN 2.196970 2.239796" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting this table out of the TRE \n", + "\n", + "- If you submit just this table to get it out of the TRE, you're going to get an email from the output checkers asking you to produce the table of counts so that they can check minimum thresholds, and maybe some other data to help them check for 'dominance' if it is financial data.\n", + "\n", + "- Defining the disclosive cells by just looking at this table can be tricky. For example, did you spot that cell on the third row and third column with a mean of 1? How many families are being aggregated to get that suspiciously integer value? Would you say that this cell is safe or not safe, and why?\n", + "\n", + "- All this to-and-fro adds effort and delays to the process of getting your output from the TRE.\n", + "\n", + "- Acro is actually designed to help you with this!\n", + "\n", + "- Because you asked for a `mean` statistic, it knows about the possible risks and so checks them for you (and the output checkers)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "### Producing tables using acro\n", + "\n", + "In the next example we will use **acro** to produce the same table. The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for.\n", "\n", - "### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categorical variables\n", "Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write\n", "````\n", "pd.crosstab(...)\n", "````\n", "so the only change is to use the prefix `acro.` rather than `pd.`\n", "\n", - "_NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing.
\n", - "\n", - "Going through the parameters in order:\n", - "- `index=df.recommendation` and `columns=df.parents` tell it we want to create a cross tabulation by two categorical variables: recommendation and parents \n", - "- setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell)\n", - "- setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*)\n", - "\n", - "Now run the next cell." + "_NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 75, "metadata": {}, "outputs": [ { @@ -392,15 +513,15 @@ " \n", " \n", " not_recom\n", - " 3.125000\n", - " 3.114583\n", - " 3.086806\n", + " 3.106250\n", + " 3.115972\n", + " 3.143056\n", " \n", " \n", " priority\n", - " 2.578089\n", - " 3.056604\n", - " 3.153846\n", + " 2.621212\n", + " 3.016846\n", + " 3.108628\n", " \n", " \n", " recommend\n", @@ -410,15 +531,15 @@ " \n", " \n", " spec_prior\n", - " 3.386746\n", - " 3.318829\n", - " 3.358839\n", + " 3.320969\n", + " 3.326741\n", + " 3.357520\n", " \n", " \n", " very_recom\n", " NaN\n", - " 2.212121\n", - " 2.198980\n", + " 2.196970\n", + " 2.239796\n", " \n", " \n", "\n", @@ -427,14 +548,14 @@ "text/plain": [ "parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.125000 3.114583 3.086806\n", - "priority 2.578089 3.056604 3.153846\n", + "not_recom 3.106250 3.115972 3.143056\n", + "priority 2.621212 3.016846 3.108628\n", "recommend NaN NaN 1.000000\n", - "spec_prior 3.386746 3.318829 3.358839\n", - "very_recom NaN 2.212121 2.198980" + "spec_prior 3.320969 3.326741 3.357520\n", + "very_recom NaN 2.196970 2.239796" ] }, - "execution_count": 5, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -450,24 +571,13 @@ "metadata": {}, "source": [ "### How to understand this output\n", - "The top part (with a pink background) is the risk analysis produced by acro. \n", + "The top part (with a pink background) is the result of the automatic risk analysis done in the background by acro.\n", "It is telling us that:\n", "- the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check\n", "- then it is showing which cells failed so you can choose how to respond\n", "- finally it is telling us that is has saved the table and risk assessment to our acro session with id \"output_0\"\n", "\n", - "The part below is the normal output produced by the pandas _crosstab()_ function.\n", - "\n", - "- In this table, it might be easy to identifying \"Obvious\" Risks. It is simple to spot that empty cells (where values are NaN) can be disclosive. If the mean is NaN, it means no records in the dataset match those specific criteria. For example, a NaN in the first column and third row indicates that there are no children whose parents are great_pret and whose recommendation is recommend. This information itself can be disclosive.\n", - "\n", - "- But it is much harder to spot the \"Hidden\" Risks (for example a cell with a mean of 10) just by looking at this table. You always need more information than what is provided in a single table. \n", - "\n", - "-In this table, you would need:\n", - " - A secondary count table to verify if the number of records in that cell falls below a required threshold. \n", - " - To assess the risk of dominance (calculate the p-ratio), you would need to extract all individual records that fall into that specific cell from the original dataset, sort them, and manually calculate the ratio to check it against the TRE (Trusted Research Environment) appetite.\n", - " - Another check for dominance is The NK-rule which is even more complicated.\n", - " \n", - " Imagine doing this for much larger tables! Actually acro can do this for us in the background!\n", + "The part below is the normal output produced by the pandas _crosstab()_ function, same as the table we produced above using Pandas.\n", "\n", "\n", "### How to respond to this input\n", @@ -495,7 +605,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -554,15 +664,15 @@ " \n", " \n", " not_recom\n", - " 3.125000\n", - " 3.114583\n", - " 3.086806\n", + " 3.106250\n", + " 3.115972\n", + " 3.143056\n", " \n", " \n", " priority\n", - " 2.578089\n", - " 3.056604\n", - " 3.153846\n", + " 2.621212\n", + " 3.016846\n", + " 3.108628\n", " \n", " \n", " recommend\n", @@ -572,15 +682,15 @@ " \n", " \n", " spec_prior\n", - " 3.386746\n", - " 3.318829\n", - " 3.358839\n", + " 3.320969\n", + " 3.326741\n", + " 3.357520\n", " \n", " \n", " very_recom\n", " NaN\n", - " 2.212121\n", - " 2.198980\n", + " 2.196970\n", + " 2.239796\n", " \n", " \n", "\n", @@ -589,14 +699,14 @@ "text/plain": [ "parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.125000 3.114583 3.086806\n", - "priority 2.578089 3.056604 3.153846\n", + "not_recom 3.106250 3.115972 3.143056\n", + "priority 2.621212 3.016846 3.108628\n", "recommend NaN NaN NaN\n", - "spec_prior 3.386746 3.318829 3.358839\n", - "very_recom NaN 2.212121 2.198980" + "spec_prior 3.320969 3.326741 3.357520\n", + "very_recom NaN 2.196970 2.239796" ] }, - "execution_count": 6, + "execution_count": 76, "metadata": {}, "output_type": "execute_result" } @@ -617,8 +727,8 @@ "Going through the parameters in order:\n", "- passing a list of variable names to `index` (rather than a single variable/column name) tells it we want a hierarchy within the rows.\n", " - we can do the same to columns as well (or instead) if we want to \n", - "- setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell)\n", - "- setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*)\n", + "- setting `values=df.children` tells it we want to report something about the number of children for each sub-group (table cell)\n", + "- setting `aggfunc=mode` tells it the statistic we want to report is the mode of children\n", "- setting `margins=True` tells it to display row and column sub-totals \n", "\n", "It's worth noting that including the totals there are 6 columns in the risk assessment and 5 in the suppressed table.
\n", @@ -627,28 +737,30 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "INFO:acro:get_summary(): review; threshold: 2 cells suppressed; p-ratio: 9 cells suppressed; nk-rule: 9 cells suppressed; \n", + "INFO:acro:If there are multiple modes, one of them is randomly selected and displayed.\n", + "INFO:acro:get_summary(): review; all-values-are-same: 9 cells suppressed; \n", "INFO:acro:outcome_df:\n", - "-----------------------------------------------------------------------------------------------------------------|\n", - "|recommendation | not_recom| priority recommend |spec_prior |very_recom |All|\n", - "|parents finance | | | | | |\n", - "-----------------------------------------------------------------------------------------------------------------|\n", - "|great_pret convenient| ok | ok p-ratio; nk-rule; | ok | p-ratio; nk-rule; | ok|\n", - "| inconv | ok | ok p-ratio; nk-rule; | ok | p-ratio; nk-rule; | ok|\n", - "|pretentious convenient| ok | ok p-ratio; nk-rule; | ok | ok | ok|\n", - "| inconv | ok | ok p-ratio; nk-rule; | ok | ok | ok|\n", - "|usual convenient| ok | ok threshold; p-ratio; nk-rule; | ok | ok | ok|\n", - "| inconv | ok | ok p-ratio; nk-rule; | ok | ok | ok|\n", - "|All | ok | ok threshold; p-ratio; nk-rule; | ok | ok | ok|\n", - "-----------------------------------------------------------------------------------------------------------------|\n", + "------------------------------------------------------------------------------------------------------------|\n", + "|recommendation | not_recom| priority recommend |spec_prior |very_recom |All|\n", + "|parents finance | | | | | |\n", + "------------------------------------------------------------------------------------------------------------|\n", + "|great_pret convenient| ok | ok all-values-are-same; | ok | all-values-are-same; | ok|\n", + "| inconv | ok | ok all-values-are-same; | ok | all-values-are-same; | ok|\n", + "|pretentious convenient| ok | ok all-values-are-same; | ok | ok | ok|\n", + "| inconv | ok | ok all-values-are-same; | ok | ok | ok|\n", + "|usual convenient| ok | ok all-values-are-same; | ok | ok | ok|\n", + "| inconv | ok | ok all-values-are-same; | ok | ok | ok|\n", + "|All | ok | ok all-values-are-same; | ok | ok | ok|\n", + "------------------------------------------------------------------------------------------------------------|\n", "\n", + "INFO:acro:If there are multiple modes, one of them is randomly selected and displayed.\n", "INFO:acro:records:add(): output_2\n", "INFO:acro:records:exception request was added to output_2\n" ] @@ -695,80 +807,80 @@ " \n", " great_pret\n", " convenient\n", - " 3.111111\n", - " 2.753906\n", - " 3.354526\n", + " 1.0\n", + " 1.0\n", + " 3.0\n", " NaN\n", - " 3.131019\n", + " 2\n", " \n", " \n", " inconv\n", - " 3.138889\n", - " 2.317919\n", - " 3.414077\n", + " 1.0\n", + " 1.0\n", + " 3.0\n", " NaN\n", - " 3.146759\n", + " 2\n", " \n", " \n", " pretentious\n", " convenient\n", - " 3.123611\n", - " 3.072917\n", - " 3.304795\n", - " 2.636364\n", - " 3.134722\n", + " 1.0\n", + " 1.0\n", + " 3.0\n", + " 1.0\n", + " 2\n", " \n", " \n", " inconv\n", - " 3.105556\n", - " 3.039106\n", - " 3.330882\n", - " 1.363636\n", - " 3.118981\n", + " 2.0\n", + " 1.0\n", + " 3.0\n", + " 1.0\n", + " 1\n", " \n", " \n", " usual\n", " convenient\n", - " 3.050000\n", - " 3.117220\n", - " 3.293605\n", - " 2.623077\n", - " 3.093142\n", + " 1.0\n", + " 1.0\n", + " 3.0\n", + " 1.0\n", + " 2\n", " \n", " \n", " inconv\n", - " 3.123611\n", - " 3.190625\n", - " 3.413043\n", - " 1.363636\n", - " 3.155093\n", + " 3.0\n", + " 3.0\n", + " 3.0\n", + " 1.0\n", + " 2\n", " \n", " \n", " All\n", " \n", - " 3.108796\n", - " 3.004219\n", - " 3.360287\n", - " 2.204268\n", - " 3.129958\n", + " 2.0\n", + " 1.0\n", + " 3.0\n", + " 1.0\n", + " 2\n", " \n", " \n", "\n", "" ], "text/plain": [ - "recommendation not_recom priority spec_prior very_recom All\n", - "parents finance \n", - "great_pret convenient 3.111111 2.753906 3.354526 NaN 3.131019\n", - " inconv 3.138889 2.317919 3.414077 NaN 3.146759\n", - "pretentious convenient 3.123611 3.072917 3.304795 2.636364 3.134722\n", - " inconv 3.105556 3.039106 3.330882 1.363636 3.118981\n", - "usual convenient 3.050000 3.117220 3.293605 2.623077 3.093142\n", - " inconv 3.123611 3.190625 3.413043 1.363636 3.155093\n", - "All 3.108796 3.004219 3.360287 2.204268 3.129958" + "recommendation not_recom priority spec_prior very_recom All\n", + "parents finance \n", + "great_pret convenient 1.0 1.0 3.0 NaN 2\n", + " inconv 1.0 1.0 3.0 NaN 2\n", + "pretentious convenient 1.0 1.0 3.0 1.0 2\n", + " inconv 2.0 1.0 3.0 1.0 1\n", + "usual convenient 1.0 1.0 3.0 1.0 2\n", + " inconv 3.0 3.0 3.0 1.0 2\n", + "All 2.0 1.0 3.0 1.0 2" ] }, - "execution_count": 7, + "execution_count": 77, "metadata": {}, "output_type": "execute_result" } @@ -779,7 +891,7 @@ " index=[df.parents, df.finance],\n", " columns=df.recommendation,\n", " values=df.children,\n", - " aggfunc=\"mean\",\n", + " aggfunc=\"mode\",\n", " margins=True,\n", ")" ] @@ -804,7 +916,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -870,7 +982,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -882,7 +994,7 @@ "type: table\n", "properties: {'method': 'crosstab'}\n", "sdc: {'summary': {'suppressed': False, 'negative': 0, 'missing': 0, 'threshold': 1, 'p-ratio': 4, 'nk-rule': 4, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 2]], 'p-ratio': [[2, 0], [2, 1], [2, 2], [4, 0]], 'nk-rule': [[2, 0], [2, 1], [2, 2], [4, 0]], 'all-values-are-same': []}}\n", - "command: acro.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")\n", + "command: acro.crosstab(\n", "summary: fail; threshold: 1 cells may need suppressing; p-ratio: 4 cells may need suppressing; nk-rule: 4 cells may need suppressing; \n", "outcome: parents great_pret pretentious \\\n", "recommendation \n", @@ -901,12 +1013,12 @@ "very_recom ok \n", "output: [parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.125000 3.114583 3.086806\n", - "priority 2.578089 3.056604 3.153846\n", + "not_recom 3.106250 3.115972 3.143056\n", + "priority 2.621212 3.016846 3.108628\n", "recommend NaN NaN 1.000000\n", - "spec_prior 3.386746 3.318829 3.358839\n", - "very_recom NaN 2.212121 2.198980]\n", - "timestamp: 2026-06-24T10:38:39.678463\n", + "spec_prior 3.320969 3.326741 3.357520\n", + "very_recom NaN 2.196970 2.239796]\n", + "timestamp: 2026-06-30T15:15:11.867068\n", "comments: []\n", "exception: \n", "\n", @@ -915,7 +1027,7 @@ "type: table\n", "properties: {'method': 'crosstab'}\n", "sdc: {'summary': {'suppressed': True, 'negative': 0, 'missing': 0, 'threshold': 1, 'p-ratio': 4, 'nk-rule': 4, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[2, 2]], 'p-ratio': [[2, 0], [2, 1], [2, 2], [4, 0]], 'nk-rule': [[2, 0], [2, 1], [2, 2], [4, 0]], 'all-values-are-same': []}}\n", - "command: acro.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")\n", + "command: acro.crosstab(\n", "summary: review; threshold: 1 cells suppressed; p-ratio: 4 cells suppressed; nk-rule: 4 cells suppressed; \n", "outcome: parents great_pret pretentious \\\n", "recommendation \n", @@ -934,12 +1046,12 @@ "very_recom ok \n", "output: [parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.125000 3.114583 3.086806\n", - "priority 2.578089 3.056604 3.153846\n", + "not_recom 3.106250 3.115972 3.143056\n", + "priority 2.621212 3.016846 3.108628\n", "recommend NaN NaN NaN\n", - "spec_prior 3.386746 3.318829 3.358839\n", - "very_recom NaN 2.212121 2.198980]\n", - "timestamp: 2026-06-24T10:38:39.772559\n", + "spec_prior 3.320969 3.326741 3.357520\n", + "very_recom NaN 2.196970 2.239796]\n", + "timestamp: 2026-06-30T15:15:11.943187\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -947,38 +1059,38 @@ "status: review\n", "type: table\n", "properties: {'method': 'crosstab'}\n", - "sdc: {'summary': {'suppressed': True, 'negative': 0, 'missing': 0, 'threshold': 2, 'p-ratio': 9, 'nk-rule': 9, 'all-values-are-same': 0}, 'cells': {'negative': [], 'missing': [], 'threshold': [[4, 2], [6, 2]], 'p-ratio': [[0, 2], [0, 4], [1, 2], [1, 4], [2, 2], [3, 2], [4, 2], [5, 2], [6, 2]], 'nk-rule': [[0, 2], [0, 4], [1, 2], [1, 4], [2, 2], [3, 2], [4, 2], [5, 2], [6, 2]], 'all-values-are-same': []}}\n", + "sdc: {'summary': {'suppressed': True, 'negative': 0, 'missing': 0, 'threshold': 0, 'p-ratio': 0, 'nk-rule': 0, 'all-values-are-same': 9}, 'cells': {'negative': [], 'missing': [], 'threshold': [], 'p-ratio': [], 'nk-rule': [], 'all-values-are-same': [[0, 2], [0, 4], [1, 2], [1, 4], [2, 2], [3, 2], [4, 2], [5, 2], [6, 2]]}}\n", "command: acro.crosstab(\n", - "summary: review; threshold: 2 cells suppressed; p-ratio: 9 cells suppressed; nk-rule: 9 cells suppressed; \n", - "outcome: recommendation not_recom priority recommend \\\n", - "parents finance \n", - "great_pret convenient ok ok p-ratio; nk-rule; \n", - " inconv ok ok p-ratio; nk-rule; \n", - "pretentious convenient ok ok p-ratio; nk-rule; \n", - " inconv ok ok p-ratio; nk-rule; \n", - "usual convenient ok ok threshold; p-ratio; nk-rule; \n", - " inconv ok ok p-ratio; nk-rule; \n", - "All ok ok threshold; p-ratio; nk-rule; \n", + "summary: review; all-values-are-same: 9 cells suppressed; \n", + "outcome: recommendation not_recom priority recommend spec_prior \\\n", + "parents finance \n", + "great_pret convenient ok ok all-values-are-same; ok \n", + " inconv ok ok all-values-are-same; ok \n", + "pretentious convenient ok ok all-values-are-same; ok \n", + " inconv ok ok all-values-are-same; ok \n", + "usual convenient ok ok all-values-are-same; ok \n", + " inconv ok ok all-values-are-same; ok \n", + "All ok ok all-values-are-same; ok \n", "\n", - "recommendation spec_prior very_recom All \n", - "parents finance \n", - "great_pret convenient ok p-ratio; nk-rule; ok \n", - " inconv ok p-ratio; nk-rule; ok \n", - "pretentious convenient ok ok ok \n", - " inconv ok ok ok \n", - "usual convenient ok ok ok \n", - " inconv ok ok ok \n", - "All ok ok ok \n", - "output: [recommendation not_recom priority spec_prior very_recom All\n", - "parents finance \n", - "great_pret convenient 3.111111 2.753906 3.354526 NaN 3.131019\n", - " inconv 3.138889 2.317919 3.414077 NaN 3.146759\n", - "pretentious convenient 3.123611 3.072917 3.304795 2.636364 3.134722\n", - " inconv 3.105556 3.039106 3.330882 1.363636 3.118981\n", - "usual convenient 3.050000 3.117220 3.293605 2.623077 3.093142\n", - " inconv 3.123611 3.190625 3.413043 1.363636 3.155093\n", - "All 3.108796 3.004219 3.360287 2.204268 3.129958]\n", - "timestamp: 2026-06-24T10:38:40.198688\n", + "recommendation very_recom All \n", + "parents finance \n", + "great_pret convenient all-values-are-same; ok \n", + " inconv all-values-are-same; ok \n", + "pretentious convenient ok ok \n", + " inconv ok ok \n", + "usual convenient ok ok \n", + " inconv ok ok \n", + "All ok ok \n", + "output: [recommendation not_recom priority spec_prior very_recom All\n", + "parents finance \n", + "great_pret convenient 1.0 1.0 3.0 NaN 2\n", + " inconv 1.0 1.0 3.0 NaN 2\n", + "pretentious convenient 1.0 1.0 3.0 1.0 2\n", + " inconv 2.0 1.0 3.0 1.0 1\n", + "usual convenient 1.0 1.0 3.0 1.0 2\n", + " inconv 3.0 3.0 3.0 1.0 2\n", + "All 2.0 1.0 3.0 1.0 2]\n", + "timestamp: 2026-06-30T15:15:12.135419\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -1002,7 +1114,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -1027,7 +1139,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 81, "metadata": {}, "outputs": [ { @@ -1056,7 +1168,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 82, "metadata": {}, "outputs": [ { @@ -1100,7 +1212,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 83, "metadata": {}, "outputs": [ { @@ -1135,7 +1247,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 84, "metadata": {}, "outputs": [ { diff --git a/notebooks/acro_demo_2026.py b/notebooks/acro_demo_2026.py index a3df73ed..0146286e 100644 --- a/notebooks/acro_demo_2026.py +++ b/notebooks/acro_demo_2026.py @@ -7,60 +7,61 @@ # # ACRO Demonstration # This is a simple notebook to get you started with using the ```acro``` package to add disclosure risk control to your analysis. -# +# # ### Assumptions -# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. +# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. # In particular the use of the industry-standard Pandas package for: # - storing and manipulating datasets # - creating basic tables, pivot_tables, and plots (e.g. histograms) -# +# # This example is a Jupyter notebook split into cells. # - Cells may contain code or text/images, and normally they are processed by stepping through them one-by-one. # - To run (or render) a cell click the *run* icon (at the top of the page) or *shift-return* on your keyboard. # That will display any output created (for a code cell) and move the focus to the next cell. +# %% [markdown] # ## A: The basic concepts -# ### 1: A research _session_: +# ### 1: A research _session_: # by which we mean the activity of running a series of commands (interactively or via a script) that: # - ingest some data, # - manipulate it, and then # - produce (and store) some outputs. -# -# ### 2: Types of commands: +# +# ### 2: Types of commands: # Whether interactive, or just running a final script, we can think of the commands that get run in a session as dividing into: # - *manipulation* commands that load and transform data into the shape you want # - *feedback* commands that report on your data - but are never intended to be exported. # For example, running ```head()``` or ```describe()``` commands to make sure your manipulations have got the data into the format you want. # - *query* commands that produce an output from your data (table/plot/regression model etc.) that you might want to export from the Trusted Research Environment (TRE) -# -# ### 3: Risk Assessment vs decision making: +# +# ### 3: Risk Assessment vs decision making: # SACRO stands for Semi-Automated Checking of Research Outputs.
-# The prefix 'Semi' is important here - because it in a principles-based system humans should make _decisions_ about output requests.
+# The prefix 'Semi' is important here - because in a principles-based system humans should make _decisions_ about output requests.
# To help with that we provide the SACRO-Viewer, which collates all the relevant information for them. -# -# A key part of that information is the _Risk Assessment_. +# +# A key part of that information is the _Risk Assessment_. # - Since it involves calculating metrics and comparing them to thresholds (the TRE's risk appetite) it can be done automatically, at the time an output query runs on the data. # - This is what the ACRO tool does when you use it as part of your workflow. -# +# # ### 4: What ACRO does -# The ACRO package aims to support you in producing *Safe Outputs* within minimal changes to your work flow. +# The ACRO package aims to support you in producing *Safe Outputs* with minimal changes to your work flow. # To do that we provide: # - drop-in replacements for the most commonly used *output commands*, # - keeping the same syntax as the originals, and -# - supporting as many of the options as we can (features supported will increase over time in response demand). +# - supporting as many of the options as we can (features supported will increase over time in response to demand). # - a set of *session-management* commands to help you manage the set of files you request for output. # - **Important to note** that currently acro outputs results (tables, details of regression models etc.) as `.csv` files.
# - In other words we separate the processes of _creating_ outputs - which *must* be done *inside* the TRE.
# from the process of _formatting_ them for publication - which can be done outside the TRE with your preferred toolchain. # - ACRO handles creation. We are interested in hearing from researchers whether it is important to support them with formatting +# %% [markdown] # ## B: Getting Started with the demonstration -# +# # ### Step 1: Setting up the environment with the tools we will use # We will begin by importing some standard data science packages, and also the acro package itself. -# In[1]: - +# %% import os import numpy as np @@ -68,38 +69,35 @@ from acro import ACRO +# %% [markdown] # ### Step 2: Starting an ACRO session -# To do this we create an acro object by running the cell below. -# +# To do this we create an acro object by running the cell below. +# # You can leave out the default parameters, but the cell below shows how you can: # - provide the name of a *config* (risk appetite) file the TRE may have asked you to use # - turn automatic suppression on or off right from the start of your session -# +# # Note that when the cell runs it should report (in a different coloured font/background) # - what version of acro is running: *this should be 0.4.12* # - the TRE's risk appetite: that defines the rules your outputs will be checked against. # - whether suppression is automatically applied to disclosive outputs. -# In[2]: - - +# %% acro = ACRO(config="default", suppress=False) - +# %% [markdown] # ### Step 3: Loading some test data -# +# # The following cells in this step just contain standard *ingestion* and *manipulation* commands to load some data into a Pandas dataframe ready to be queried.
# We will use some open-source data about nursery admissions. -# -# **There is no change to your workflow here** -# - Do whatever you want in this step! -# - We just assume you end up with your data in a panda dataframe. -# -# - -# In[3]: - - +# +# **There is no change to your workflow here** +# - Do whatever you want in this step! +# - We just assume you end up with your data in a pandas dataframe. +# +# + +# %% from scipy.io.arff import loadarff ##--- Manipulation commands --- @@ -130,128 +128,135 @@ axis=1, ) - -# In[4]: - - +# %% ##--- Feedback Command ---- # show the first 5 rows to make sure everything is how we would expect df.head() - +# %% [markdown] # ## C: Producing tables that are 'Safe Outputs' -# -# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. +# +# ### Producing tables using standard python libraries +# +# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. # - There are hundreds (thousands?) of web sites showing how to do this. -# - You can make (hierarchical) two-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row) +# - You can make (hierarchical) 2-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row) # - you can specify what the table cells contain by: # - providing a statistic - for example: mean, count, std deviation, median etc.(pandas calls these *aggregation functions*) # - specifying what variable to report on -# -# The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for -# -# ### Example 1: A 2-D table calculating the mean of a target variable, grouped by two categorical variables -# +# +# In the next example: +# - We will use the Pandas *crosstab()* function to create a summary table. +# +# - The table will calculate the **mean number of children** for every combination of **Recommendation** and **Parental status**. Going through the parameters in order: +# - `index=df.recommendation`: Defines the rows of the table using the recommendation column from the dataset. +# +# - `columns=df.parents`: Defines the columns of the table using the parents column from the dataset. +# - `values=df.children`: Specifies that we want to perform calculations on the children (the name of a column in the dataset). +# +# - `aggfunc="mean"`: Tells the function to calculate the **average (mean)** for those values. + +# %% +pd.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean") + +# %% [markdown] +# ### Getting this table out of the TRE +# +# - If you submit just this table to get it out of the TRE, you're going to get an email from the output checkers asking you to produce the table of counts so that they can check minimum thresholds, and maybe some other data to help them check for 'dominance' if it is financial data. +# +# - Defining the disclosive cells by just looking at this table can be tricky. For example, did you spot that cell on the third row and third column with a mean of 1? How many families are being aggregated to get that suspiciously integer value? Would you say that this cell is safe or not safe, and why? +# +# - All this to-and-fro adds effort and delays to the process of getting your output from the TRE. +# +# - Acro is actually designed to help you with this! +# +# - Because you asked for a `mean` statistic, it knows about the possible risks and so checks them for you (and the output checkers). + +# %% [markdown] +# +# ### Producing tables using acro +# +# In the next example we will use **acro** to produce the same table. The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for. +# # Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write # ```` # pd.crosstab(...) # ```` # so the only change is to use the prefix `acro.` rather than `pd.` -# +# # _NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing. -# -# Going through the parameters in order: -# - `index=df.recommendation` and `columns=df.parents` tell it we want to create a cross tabulation by two categorical variables: recommendation and parents -# - setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell) -# - setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*) -# -# Now run the next cell. - -# In[5]: - +# %% acro.crosstab( index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" ) - +# %% [markdown] # ### How to understand this output -# The top part (with a pink background) is the risk analysis produced by acro. +# The top part (with a pink background) is the result of the automatic risk analysis done in the background by acro. # It is telling us that: # - the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check # - then it is showing which cells failed so you can choose how to respond # - finally it is telling us that is has saved the table and risk assessment to our acro session with id "output_0" -# -# The part below is the normal output produced by the pandas _crosstab()_ function. -# - In this table, it might be easy to identifying "Obvious" Risks. It is simple to spot that empty cells (where values are NaN) can be disclosive. If the mean is NaN, it means no records in the dataset match those specific criteria. For example, a NaN in the first column and third row indicates that there are no children whose parents are great_pret and whose recommendation is recommend. This information itself can be disclosive. - -# - But it is much harder to spot the "Hidden" Risks (for example a cell with a mean of 10) just by looking at this table. You always need more information than what is provided in a single table. - -# -In this table, you would need: -# - A secondary count table to verify if the number of records in that cell falls below a required threshold. -# - To assess the risk of dominance (calculate the p-ratio), you would need to extract all individual records that fall into that specific cell from the original dataset, sort them, and manually calculate the ratio to check it against the TRE (Trusted Research Environment) appetite. -# - Another check for dominance is The NK-rule which is even more complicated. -# -# Imagine doing this for much larger tables! Actually acro can do this for us in the background! -# +# +# The part below is the normal output produced by the pandas _crosstab()_ function, same as the table we produced above using Pandas. +# +# # ### How to respond to this input # There are basically three choices: # 1. We might decide these low numbers reveal something where the public interest outweighs the disclosure risk.
# Rather than being a strict rules-based system, acro lets you attach an 'exception request' to an output, to send a message to the output checkers.
-# For example, you could type: +# For example, you could type: # ```` # acro.add_exception('output_0',"I think you should let me have this because...") # ```` -# +# # 2. We redesign our data so that table so that none of the cells in the resulting table represent fewer than _n_ people (10 for the default risk appetite)
# For example, we could recode _'very_recommend'_ and _'priority'_ into one label.
# But maybe it is revealing that the _'recommend'_ label is not used? -# +# # 3. We can redact the disclosive cells - and **acro will do this for us**.
# We simply enable the option to suppress disclosive cells and re-run the query. -# -# The cell below shows option 3. +# +# The cell below shows option 3.b # When you run the cell below you should see that: # - the status now changes to `review` (so the output-checker knows what has been applied) # - the code automatically adds an exception request saying that suppression has been applied -# - and, most importantly, the cells are redacted. - -# In[6]: - +# - and, most importantly, the cells are redacted. +# %% acro.enable_suppression() acro.crosstab( index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" ) +# %% [markdown] # ## An example of a more complex table -# Just to show off the sort of tables that `cross_tab()` can produce, let's make something more complex.
+# Just to show off the sort of tables that `crosstab()` can produce, let's make something more complex.
# Going through the parameters in order: # - passing a list of variable names to `index` (rather than a single variable/column name) tells it we want a hierarchy within the rows. -# - we can do the same to columns as well (or instead) if we want to -# - setting `values=df.children`(the name of a column in the dataset) tells it we want to report something about the number of children for each sub-group (table cell) -# - setting `aggfunc=mean` tells it the statistic we want to report is the mean number of children (which introduces additional risks of *dominance*) -# - setting `margins=True` tells it to display row and column sub-totals -# +# - we can do the same to columns as well (or instead) if we want to +# - setting `values=df.children` tells it we want to report something about the number of children for each sub-group (table cell) +# - setting `aggfunc=mode` tells it the statistic we want to report is the mode of children +# - setting `margins=True` tells it to display row and column sub-totals +# # It's worth noting that including the totals there are 6 columns in the risk assessment and 5 in the suppressed table.
# This is because after suppression has replaced numbers with `NaN`, pandas removes the fully suppressed column (_'recommend'_) from the table. -# In[7]: - - +# %% acro.suppress = True acro.crosstab( index=[df.parents, df.finance], columns=df.recommendation, values=df.children, - aggfunc="mean", + aggfunc="mode", margins=True, ) - +# %% [markdown] # ## D: What other sorts of analysis does ACRO currently support? # We are continually adding support for more types of analysis as users prioritise them. -# +# # ACRO currently supports: # - **Tables** via `acro.crosstab()` and `acro.pivot_table()`. # - supported aggregation functions are: _mean_, _median_, _sum_, _std_, _count_, and _mode_.
@@ -259,21 +264,19 @@ # - **Histograms** via:`acro.hist()`
# - **Regression** via: `acro.ols()`, `acro.logit()`,`acro.probit()` # with options for specifying formula in 'R-style' by adding the suffix 'r' e.g. `acro.olsr()` etc. -# +# # You can get help on using any of these using the standard python `help()` syntax as shown in the next cell -# In[8]: - - +# %% help(acro.logit) - +# %% [markdown] # ## E: ACRO functionality to let users manage their outputs # As explained above, you need to create an "acro session" whenever your code is run. -# +# # After that, every time you run an acro `query' command both the output and the risk assessment are saved as part of the acro session. -# -# But we recognise that: +# +# But we recognise that: # - You may not want to request release of all your outputs - for example, the first table we produced above. # - It is good practice to provide a more informative name than just *output_n* for the .csv files that acro produces # - It helps the output checker if you provide some comments saying what the outputs are. @@ -281,98 +284,90 @@ # - outputs from analyses that acro doesn't currently support # - your code itself (which many journals want) # - maybe a version of your paper in pdf/word format etc. -# +# # Therefore acro provides the following commands for 'session management' # ### 1: Listing the current contents of an ACRO session -# This output is not beautiful (there's a GUI come soon) but should let you identify outputs you want to rename,comment on, or delete - -# In[9]: - +# This output is not beautiful (there's a GUI coming soon) but it should let you identify outputs you want to rename,comment on, or delete. +# %% _ = acro.print_outputs() - -# ### 2: Remove some ACRO outputs before finalising +# %% [markdown] +# ### 2: Remove some ACRO outputs before finalising # At the start of this demo we made a disclosive output -it;s the first one with status _fail_. -# +# # We don't want to waste the output checker's time so lets remove it. -# In[10]: - - +# %% acro.remove_output("output_0") - +# %% [markdown] # ### 3: Rename ACRO outputs before finalising # This is an example of renaming the outputs to provide more descriptive names. -# In[11]: - - +# %% acro.rename_output("output_1", " crosstab_recommendation_vs_parents") acro.rename_output("output_2", "mean_children_by_parents_finance_recommendation") - +# %% [markdown] # ### 4: Add a comment to output -# This is an example of adding a comment to outputs. +# This is an example of adding a comment to outputs. # It can be used to provide a description or to pass additional information to the TRE staff.
# They will see it alongside your file in the output checking viewer - rather than having it in an email somewhere. -# In[12]: - - +# %% acro.add_comments( "mean_children_by_parents_finance_recommendation", "too few cases of recommend to report", ) - +# %% [markdown] # ### 5. Request an exception # An example of providing a reason why an exception should be made # ```` # acro.add_exception("output_n", "This is evidence of systematic bias?") # ```` +# %% [markdown] # ### 6: Adding a custom output. -# +# # As mentioned above you might want to request release of all sorts of things # - including your code, # - or outputs from analyses *acro* doesn't support (yet) -# +# # In ACRO we can add a file to our session with a comment describing what it is -# In[13]: - - +# %% acro.custom_output("acro_demo_2026.py", "This is the code that produced this session") - +# %% [markdown] # ## F: Finishing your session and producing a folder of files to release. -# This is an example of the function _finalise()_ which the users must call at the end of each session. +# This is an example of the function _finalise()_ which the users must call at the end of each session. # - It takes each output and saves it to a CSV file (or the original file type for custom outputs) # - It also saves the SDC analysis for each output to a json file. # - It adds checksums for everything - so we know they've not been edited. # - It puts them all in a folder with the name you supply. -# +# # **ACRO will not overwrite previous sessions** -# +# # So every time you call finalise on a session you need to either: # - delete the previous folder, or -# - provide a new folder name - -# In[14]: - +# - provide a new folder name +# %% output = acro.finalise("my_acro_outputs_v1") - +# %% [markdown] # ## G: Reminder about getting help while you work -# +# # - if you remember the name of the command and want an explanation or to explain the syntax
# from the python prompt type: ` help(acro.command_name)` -# -# +# +# # - if you can't remember the name of the command, from the python prompt type: `help(acro.ACRO)` -# - not as user friendly but will list all the available commands +# - not as user friendly but will list all the available commands + +# %% + + -# In[ ]: From 66ee28c805ab6bc6ebdbbcdd9d97713c5af5b3b1 Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Tue, 30 Jun 2026 15:18:42 +0100 Subject: [PATCH 4/6] updating notebooks --- notebooks/acro_demo_2026.ipynb | 4 +- notebooks/acro_demo_2026.py | 145 ++++++++++++++++----------------- 2 files changed, 75 insertions(+), 74 deletions(-) diff --git a/notebooks/acro_demo_2026.ipynb b/notebooks/acro_demo_2026.ipynb index 909a8c75..e0dc03d7 100644 --- a/notebooks/acro_demo_2026.ipynb +++ b/notebooks/acro_demo_2026.ipynb @@ -415,7 +415,9 @@ } ], "source": [ - "pd.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\")" + "pd.crosstab(\n", + " index=df.recommendation, columns=df.parents, values=df.children, aggfunc=\"mean\"\n", + ")" ] }, { diff --git a/notebooks/acro_demo_2026.py b/notebooks/acro_demo_2026.py index 0146286e..5b62e94b 100644 --- a/notebooks/acro_demo_2026.py +++ b/notebooks/acro_demo_2026.py @@ -7,13 +7,13 @@ # # ACRO Demonstration # This is a simple notebook to get you started with using the ```acro``` package to add disclosure risk control to your analysis. -# +# # ### Assumptions -# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. +# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. # In particular the use of the industry-standard Pandas package for: # - storing and manipulating datasets # - creating basic tables, pivot_tables, and plots (e.g. histograms) -# +# # This example is a Jupyter notebook split into cells. # - Cells may contain code or text/images, and normally they are processed by stepping through them one-by-one. # - To run (or render) a cell click the *run* icon (at the top of the page) or *shift-return* on your keyboard. @@ -21,28 +21,28 @@ # %% [markdown] # ## A: The basic concepts -# ### 1: A research _session_: +# ### 1: A research _session_: # by which we mean the activity of running a series of commands (interactively or via a script) that: # - ingest some data, # - manipulate it, and then # - produce (and store) some outputs. -# -# ### 2: Types of commands: +# +# ### 2: Types of commands: # Whether interactive, or just running a final script, we can think of the commands that get run in a session as dividing into: # - *manipulation* commands that load and transform data into the shape you want # - *feedback* commands that report on your data - but are never intended to be exported. # For example, running ```head()``` or ```describe()``` commands to make sure your manipulations have got the data into the format you want. # - *query* commands that produce an output from your data (table/plot/regression model etc.) that you might want to export from the Trusted Research Environment (TRE) -# -# ### 3: Risk Assessment vs decision making: +# +# ### 3: Risk Assessment vs decision making: # SACRO stands for Semi-Automated Checking of Research Outputs.
# The prefix 'Semi' is important here - because in a principles-based system humans should make _decisions_ about output requests.
# To help with that we provide the SACRO-Viewer, which collates all the relevant information for them. -# -# A key part of that information is the _Risk Assessment_. +# +# A key part of that information is the _Risk Assessment_. # - Since it involves calculating metrics and comparing them to thresholds (the TRE's risk appetite) it can be done automatically, at the time an output query runs on the data. # - This is what the ACRO tool does when you use it as part of your workflow. -# +# # ### 4: What ACRO does # The ACRO package aims to support you in producing *Safe Outputs* with minimal changes to your work flow. # To do that we provide: @@ -57,7 +57,7 @@ # %% [markdown] # ## B: Getting Started with the demonstration -# +# # ### Step 1: Setting up the environment with the tools we will use # We will begin by importing some standard data science packages, and also the acro package itself. @@ -71,12 +71,12 @@ # %% [markdown] # ### Step 2: Starting an ACRO session -# To do this we create an acro object by running the cell below. -# +# To do this we create an acro object by running the cell below. +# # You can leave out the default parameters, but the cell below shows how you can: # - provide the name of a *config* (risk appetite) file the TRE may have asked you to use # - turn automatic suppression on or off right from the start of your session -# +# # Note that when the cell runs it should report (in a different coloured font/background) # - what version of acro is running: *this should be 0.4.12* # - the TRE's risk appetite: that defines the rules your outputs will be checked against. @@ -87,15 +87,15 @@ # %% [markdown] # ### Step 3: Loading some test data -# +# # The following cells in this step just contain standard *ingestion* and *manipulation* commands to load some data into a Pandas dataframe ready to be queried.
# We will use some open-source data about nursery admissions. -# -# **There is no change to your workflow here** -# - Do whatever you want in this step! -# - We just assume you end up with your data in a pandas dataframe. -# -# +# +# **There is no change to your workflow here** +# - Do whatever you want in this step! +# - We just assume you end up with your data in a pandas dataframe. +# +# # %% from scipy.io.arff import loadarff @@ -135,55 +135,57 @@ # %% [markdown] # ## C: Producing tables that are 'Safe Outputs' -# +# # ### Producing tables using standard python libraries -# -# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. +# +# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. # - There are hundreds (thousands?) of web sites showing how to do this. # - You can make (hierarchical) 2-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row) # - you can specify what the table cells contain by: # - providing a statistic - for example: mean, count, std deviation, median etc.(pandas calls these *aggregation functions*) # - specifying what variable to report on -# +# # In the next example: # - We will use the Pandas *crosstab()* function to create a summary table. -# +# # - The table will calculate the **mean number of children** for every combination of **Recommendation** and **Parental status**. Going through the parameters in order: # - `index=df.recommendation`: Defines the rows of the table using the recommendation column from the dataset. -# +# # - `columns=df.parents`: Defines the columns of the table using the parents column from the dataset. # - `values=df.children`: Specifies that we want to perform calculations on the children (the name of a column in the dataset). -# +# # - `aggfunc="mean"`: Tells the function to calculate the **average (mean)** for those values. # %% -pd.crosstab(index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean") +pd.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) # %% [markdown] -# ### Getting this table out of the TRE -# +# ### Getting this table out of the TRE +# # - If you submit just this table to get it out of the TRE, you're going to get an email from the output checkers asking you to produce the table of counts so that they can check minimum thresholds, and maybe some other data to help them check for 'dominance' if it is financial data. -# +# # - Defining the disclosive cells by just looking at this table can be tricky. For example, did you spot that cell on the third row and third column with a mean of 1? How many families are being aggregated to get that suspiciously integer value? Would you say that this cell is safe or not safe, and why? -# +# # - All this to-and-fro adds effort and delays to the process of getting your output from the TRE. -# +# # - Acro is actually designed to help you with this! -# +# # - Because you asked for a `mean` statistic, it knows about the possible risks and so checks them for you (and the output checkers). # %% [markdown] -# +# # ### Producing tables using acro -# +# # In the next example we will use **acro** to produce the same table. The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for. -# +# # Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write # ```` # pd.crosstab(...) # ```` # so the only change is to use the prefix `acro.` rather than `pd.` -# +# # _NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing. # %% @@ -198,31 +200,31 @@ # - the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check # - then it is showing which cells failed so you can choose how to respond # - finally it is telling us that is has saved the table and risk assessment to our acro session with id "output_0" -# +# # The part below is the normal output produced by the pandas _crosstab()_ function, same as the table we produced above using Pandas. -# -# +# +# # ### How to respond to this input # There are basically three choices: # 1. We might decide these low numbers reveal something where the public interest outweighs the disclosure risk.
# Rather than being a strict rules-based system, acro lets you attach an 'exception request' to an output, to send a message to the output checkers.
-# For example, you could type: +# For example, you could type: # ```` # acro.add_exception('output_0',"I think you should let me have this because...") # ```` -# +# # 2. We redesign our data so that table so that none of the cells in the resulting table represent fewer than _n_ people (10 for the default risk appetite)
# For example, we could recode _'very_recommend'_ and _'priority'_ into one label.
# But maybe it is revealing that the _'recommend'_ label is not used? -# +# # 3. We can redact the disclosive cells - and **acro will do this for us**.
# We simply enable the option to suppress disclosive cells and re-run the query. -# +# # The cell below shows option 3.b # When you run the cell below you should see that: # - the status now changes to `review` (so the output-checker knows what has been applied) # - the code automatically adds an exception request saying that suppression has been applied -# - and, most importantly, the cells are redacted. +# - and, most importantly, the cells are redacted. # %% acro.enable_suppression() @@ -235,11 +237,11 @@ # Just to show off the sort of tables that `crosstab()` can produce, let's make something more complex.
# Going through the parameters in order: # - passing a list of variable names to `index` (rather than a single variable/column name) tells it we want a hierarchy within the rows. -# - we can do the same to columns as well (or instead) if we want to +# - we can do the same to columns as well (or instead) if we want to # - setting `values=df.children` tells it we want to report something about the number of children for each sub-group (table cell) # - setting `aggfunc=mode` tells it the statistic we want to report is the mode of children -# - setting `margins=True` tells it to display row and column sub-totals -# +# - setting `margins=True` tells it to display row and column sub-totals +# # It's worth noting that including the totals there are 6 columns in the risk assessment and 5 in the suppressed table.
# This is because after suppression has replaced numbers with `NaN`, pandas removes the fully suppressed column (_'recommend'_) from the table. @@ -256,7 +258,7 @@ # %% [markdown] # ## D: What other sorts of analysis does ACRO currently support? # We are continually adding support for more types of analysis as users prioritise them. -# +# # ACRO currently supports: # - **Tables** via `acro.crosstab()` and `acro.pivot_table()`. # - supported aggregation functions are: _mean_, _median_, _sum_, _std_, _count_, and _mode_.
@@ -264,7 +266,7 @@ # - **Histograms** via:`acro.hist()`
# - **Regression** via: `acro.ols()`, `acro.logit()`,`acro.probit()` # with options for specifying formula in 'R-style' by adding the suffix 'r' e.g. `acro.olsr()` etc. -# +# # You can get help on using any of these using the standard python `help()` syntax as shown in the next cell # %% @@ -273,10 +275,10 @@ # %% [markdown] # ## E: ACRO functionality to let users manage their outputs # As explained above, you need to create an "acro session" whenever your code is run. -# +# # After that, every time you run an acro `query' command both the output and the risk assessment are saved as part of the acro session. -# -# But we recognise that: +# +# But we recognise that: # - You may not want to request release of all your outputs - for example, the first table we produced above. # - It is good practice to provide a more informative name than just *output_n* for the .csv files that acro produces # - It helps the output checker if you provide some comments saying what the outputs are. @@ -284,7 +286,7 @@ # - outputs from analyses that acro doesn't currently support # - your code itself (which many journals want) # - maybe a version of your paper in pdf/word format etc. -# +# # Therefore acro provides the following commands for 'session management' # ### 1: Listing the current contents of an ACRO session # This output is not beautiful (there's a GUI coming soon) but it should let you identify outputs you want to rename,comment on, or delete. @@ -293,9 +295,9 @@ _ = acro.print_outputs() # %% [markdown] -# ### 2: Remove some ACRO outputs before finalising +# ### 2: Remove some ACRO outputs before finalising # At the start of this demo we made a disclosive output -it;s the first one with status _fail_. -# +# # We don't want to waste the output checker's time so lets remove it. # %% @@ -311,7 +313,7 @@ # %% [markdown] # ### 4: Add a comment to output -# This is an example of adding a comment to outputs. +# This is an example of adding a comment to outputs. # It can be used to provide a description or to pass additional information to the TRE staff.
# They will see it alongside your file in the output checking viewer - rather than having it in an email somewhere. @@ -330,11 +332,11 @@ # %% [markdown] # ### 6: Adding a custom output. -# +# # As mentioned above you might want to request release of all sorts of things # - including your code, # - or outputs from analyses *acro* doesn't support (yet) -# +# # In ACRO we can add a file to our session with a comment describing what it is # %% @@ -342,32 +344,29 @@ # %% [markdown] # ## F: Finishing your session and producing a folder of files to release. -# This is an example of the function _finalise()_ which the users must call at the end of each session. +# This is an example of the function _finalise()_ which the users must call at the end of each session. # - It takes each output and saves it to a CSV file (or the original file type for custom outputs) # - It also saves the SDC analysis for each output to a json file. # - It adds checksums for everything - so we know they've not been edited. # - It puts them all in a folder with the name you supply. -# +# # **ACRO will not overwrite previous sessions** -# +# # So every time you call finalise on a session you need to either: # - delete the previous folder, or -# - provide a new folder name +# - provide a new folder name # %% output = acro.finalise("my_acro_outputs_v1") # %% [markdown] # ## G: Reminder about getting help while you work -# +# # - if you remember the name of the command and want an explanation or to explain the syntax
# from the python prompt type: ` help(acro.command_name)` -# -# +# +# # - if you can't remember the name of the command, from the python prompt type: `help(acro.ACRO)` -# - not as user friendly but will list all the available commands +# - not as user friendly but will list all the available commands # %% - - - From 5c4c9868e212a0fc57aea91aa47aa56b323e2c4c Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Tue, 30 Jun 2026 15:20:07 +0100 Subject: [PATCH 5/6] updating notebooks --- notebooks/acro_demo_2026.ipynb | 192 ++++----- .../ crosstab_recommendation_vs_parents_0.csv | 6 + .../my_acro_outputs_v1/acro_demo_2026.py | 372 ++++++++++++++++++ ...osstab_recommendation_vs_parents_0.csv.txt | 1 + .../checksums/acro_demo_2026.py.txt | 1 + ...y_parents_finance_recommendation_0.csv.txt | 1 + .../checksums/results.json.txt | 1 + notebooks/my_acro_outputs_v1/config.json | 10 + ...en_by_parents_finance_recommendation_0.csv | 8 + notebooks/my_acro_outputs_v1/results.json | 256 ++++++++++++ 10 files changed, 752 insertions(+), 96 deletions(-) create mode 100644 notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv create mode 100644 notebooks/my_acro_outputs_v1/acro_demo_2026.py create mode 100644 notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt create mode 100644 notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt create mode 100644 notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt create mode 100644 notebooks/my_acro_outputs_v1/checksums/results.json.txt create mode 100644 notebooks/my_acro_outputs_v1/config.json create mode 100644 notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv create mode 100644 notebooks/my_acro_outputs_v1/results.json diff --git a/notebooks/acro_demo_2026.ipynb b/notebooks/acro_demo_2026.ipynb index e0dc03d7..be23447f 100644 --- a/notebooks/acro_demo_2026.ipynb +++ b/notebooks/acro_demo_2026.ipynb @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -136,7 +136,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -289,7 +289,7 @@ "4 priority priority " ] }, - "execution_count": 73, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -329,7 +329,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -367,15 +367,15 @@ " \n", " \n", " not_recom\n", - " 3.106250\n", - " 3.115972\n", - " 3.143056\n", + " 3.100694\n", + " 3.132639\n", + " 3.130556\n", " \n", " \n", " priority\n", - " 2.621212\n", - " 3.016846\n", - " 3.108628\n", + " 2.625874\n", + " 3.039757\n", + " 3.123701\n", " \n", " \n", " recommend\n", @@ -385,15 +385,15 @@ " \n", " \n", " spec_prior\n", - " 3.320969\n", - " 3.326741\n", - " 3.357520\n", + " 3.347181\n", + " 3.375791\n", + " 3.401055\n", " \n", " \n", " very_recom\n", " NaN\n", - " 2.196970\n", - " 2.239796\n", + " 2.098485\n", + " 2.250000\n", " \n", " \n", "\n", @@ -402,14 +402,14 @@ "text/plain": [ "parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.106250 3.115972 3.143056\n", - "priority 2.621212 3.016846 3.108628\n", + "not_recom 3.100694 3.132639 3.130556\n", + "priority 2.625874 3.039757 3.123701\n", "recommend NaN NaN 1.000000\n", - "spec_prior 3.320969 3.326741 3.357520\n", - "very_recom NaN 2.196970 2.239796" + "spec_prior 3.347181 3.375791 3.401055\n", + "very_recom NaN 2.098485 2.250000" ] }, - "execution_count": 74, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -457,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -515,15 +515,15 @@ " \n", " \n", " not_recom\n", - " 3.106250\n", - " 3.115972\n", - " 3.143056\n", + " 3.100694\n", + " 3.132639\n", + " 3.130556\n", " \n", " \n", " priority\n", - " 2.621212\n", - " 3.016846\n", - " 3.108628\n", + " 2.625874\n", + " 3.039757\n", + " 3.123701\n", " \n", " \n", " recommend\n", @@ -533,15 +533,15 @@ " \n", " \n", " spec_prior\n", - " 3.320969\n", - " 3.326741\n", - " 3.357520\n", + " 3.347181\n", + " 3.375791\n", + " 3.401055\n", " \n", " \n", " very_recom\n", " NaN\n", - " 2.196970\n", - " 2.239796\n", + " 2.098485\n", + " 2.250000\n", " \n", " \n", "\n", @@ -550,14 +550,14 @@ "text/plain": [ "parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.106250 3.115972 3.143056\n", - "priority 2.621212 3.016846 3.108628\n", + "not_recom 3.100694 3.132639 3.130556\n", + "priority 2.625874 3.039757 3.123701\n", "recommend NaN NaN 1.000000\n", - "spec_prior 3.320969 3.326741 3.357520\n", - "very_recom NaN 2.196970 2.239796" + "spec_prior 3.347181 3.375791 3.401055\n", + "very_recom NaN 2.098485 2.250000" ] }, - "execution_count": 75, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -607,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -666,15 +666,15 @@ " \n", " \n", " not_recom\n", - " 3.106250\n", - " 3.115972\n", - " 3.143056\n", + " 3.100694\n", + " 3.132639\n", + " 3.130556\n", " \n", " \n", " priority\n", - " 2.621212\n", - " 3.016846\n", - " 3.108628\n", + " 2.625874\n", + " 3.039757\n", + " 3.123701\n", " \n", " \n", " recommend\n", @@ -684,15 +684,15 @@ " \n", " \n", " spec_prior\n", - " 3.320969\n", - " 3.326741\n", - " 3.357520\n", + " 3.347181\n", + " 3.375791\n", + " 3.401055\n", " \n", " \n", " very_recom\n", " NaN\n", - " 2.196970\n", - " 2.239796\n", + " 2.098485\n", + " 2.250000\n", " \n", " \n", "\n", @@ -701,14 +701,14 @@ "text/plain": [ "parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.106250 3.115972 3.143056\n", - "priority 2.621212 3.016846 3.108628\n", + "not_recom 3.100694 3.132639 3.130556\n", + "priority 2.625874 3.039757 3.123701\n", "recommend NaN NaN NaN\n", - "spec_prior 3.320969 3.326741 3.357520\n", - "very_recom NaN 2.196970 2.239796" + "spec_prior 3.347181 3.375791 3.401055\n", + "very_recom NaN 2.098485 2.250000" ] }, - "execution_count": 76, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -739,7 +739,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -809,28 +809,28 @@ " \n", " great_pret\n", " convenient\n", - " 1.0\n", + " 3.0\n", " 1.0\n", " 3.0\n", " NaN\n", - " 2\n", + " 1\n", " \n", " \n", " inconv\n", - " 1.0\n", + " 2.0\n", " 1.0\n", " 3.0\n", " NaN\n", - " 2\n", + " 1\n", " \n", " \n", " pretentious\n", " convenient\n", - " 1.0\n", + " 2.0\n", " 1.0\n", " 3.0\n", " 1.0\n", - " 2\n", + " 1\n", " \n", " \n", " inconv\n", @@ -851,20 +851,20 @@ " \n", " \n", " inconv\n", - " 3.0\n", + " 1.0\n", " 3.0\n", " 3.0\n", " 1.0\n", - " 2\n", + " 1\n", " \n", " \n", " All\n", " \n", - " 2.0\n", + " 1.0\n", " 1.0\n", " 3.0\n", " 1.0\n", - " 2\n", + " 3\n", " \n", " \n", "\n", @@ -873,16 +873,16 @@ "text/plain": [ "recommendation not_recom priority spec_prior very_recom All\n", "parents finance \n", - "great_pret convenient 1.0 1.0 3.0 NaN 2\n", - " inconv 1.0 1.0 3.0 NaN 2\n", - "pretentious convenient 1.0 1.0 3.0 1.0 2\n", + "great_pret convenient 3.0 1.0 3.0 NaN 1\n", + " inconv 2.0 1.0 3.0 NaN 1\n", + "pretentious convenient 2.0 1.0 3.0 1.0 1\n", " inconv 2.0 1.0 3.0 1.0 1\n", "usual convenient 1.0 1.0 3.0 1.0 2\n", - " inconv 3.0 3.0 3.0 1.0 2\n", - "All 2.0 1.0 3.0 1.0 2" + " inconv 1.0 3.0 3.0 1.0 1\n", + "All 1.0 1.0 3.0 1.0 3" ] }, - "execution_count": 77, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -918,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -984,7 +984,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -1015,12 +1015,12 @@ "very_recom ok \n", "output: [parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.106250 3.115972 3.143056\n", - "priority 2.621212 3.016846 3.108628\n", + "not_recom 3.100694 3.132639 3.130556\n", + "priority 2.625874 3.039757 3.123701\n", "recommend NaN NaN 1.000000\n", - "spec_prior 3.320969 3.326741 3.357520\n", - "very_recom NaN 2.196970 2.239796]\n", - "timestamp: 2026-06-30T15:15:11.867068\n", + "spec_prior 3.347181 3.375791 3.401055\n", + "very_recom NaN 2.098485 2.250000]\n", + "timestamp: 2026-06-30T15:19:49.512223\n", "comments: []\n", "exception: \n", "\n", @@ -1048,12 +1048,12 @@ "very_recom ok \n", "output: [parents great_pret pretentious usual\n", "recommendation \n", - "not_recom 3.106250 3.115972 3.143056\n", - "priority 2.621212 3.016846 3.108628\n", + "not_recom 3.100694 3.132639 3.130556\n", + "priority 2.625874 3.039757 3.123701\n", "recommend NaN NaN NaN\n", - "spec_prior 3.320969 3.326741 3.357520\n", - "very_recom NaN 2.196970 2.239796]\n", - "timestamp: 2026-06-30T15:15:11.943187\n", + "spec_prior 3.347181 3.375791 3.401055\n", + "very_recom NaN 2.098485 2.250000]\n", + "timestamp: 2026-06-30T15:19:49.603861\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -1085,14 +1085,14 @@ "All ok ok \n", "output: [recommendation not_recom priority spec_prior very_recom All\n", "parents finance \n", - "great_pret convenient 1.0 1.0 3.0 NaN 2\n", - " inconv 1.0 1.0 3.0 NaN 2\n", - "pretentious convenient 1.0 1.0 3.0 1.0 2\n", + "great_pret convenient 3.0 1.0 3.0 NaN 1\n", + " inconv 2.0 1.0 3.0 NaN 1\n", + "pretentious convenient 2.0 1.0 3.0 1.0 1\n", " inconv 2.0 1.0 3.0 1.0 1\n", "usual convenient 1.0 1.0 3.0 1.0 2\n", - " inconv 3.0 3.0 3.0 1.0 2\n", - "All 2.0 1.0 3.0 1.0 2]\n", - "timestamp: 2026-06-30T15:15:12.135419\n", + " inconv 1.0 3.0 3.0 1.0 1\n", + "All 1.0 1.0 3.0 1.0 3]\n", + "timestamp: 2026-06-30T15:19:49.815485\n", "comments: []\n", "exception: Suppression automatically applied where needed\n", "\n", @@ -1116,7 +1116,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -1141,7 +1141,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -1170,7 +1170,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -1214,7 +1214,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -1249,7 +1249,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 15, "metadata": {}, "outputs": [ { diff --git a/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv b/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv new file mode 100644 index 00000000..07ed46fa --- /dev/null +++ b/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv @@ -0,0 +1,6 @@ +recommendation,great_pret,pretentious,usual +not_recom,3.1006944444444446,3.1326388888888888,3.1305555555555555 +priority,2.625874125874126,3.039757412398922,3.1237006237006235 +recommend,,, +spec_prior,3.3471810089020773,3.3757911392405062,3.401055408970976 +very_recom,,2.0984848484848486,2.25 diff --git a/notebooks/my_acro_outputs_v1/acro_demo_2026.py b/notebooks/my_acro_outputs_v1/acro_demo_2026.py new file mode 100644 index 00000000..5b62e94b --- /dev/null +++ b/notebooks/my_acro_outputs_v1/acro_demo_2026.py @@ -0,0 +1,372 @@ +""" +Acro_demo_2026.py. + +Created (except this docstring) from acro_demo_2026.iynb +author Jim Smith. +""" + +# # ACRO Demonstration +# This is a simple notebook to get you started with using the ```acro``` package to add disclosure risk control to your analysis. +# +# ### Assumptions +# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. +# In particular the use of the industry-standard Pandas package for: +# - storing and manipulating datasets +# - creating basic tables, pivot_tables, and plots (e.g. histograms) +# +# This example is a Jupyter notebook split into cells. +# - Cells may contain code or text/images, and normally they are processed by stepping through them one-by-one. +# - To run (or render) a cell click the *run* icon (at the top of the page) or *shift-return* on your keyboard. +# That will display any output created (for a code cell) and move the focus to the next cell. + +# %% [markdown] +# ## A: The basic concepts +# ### 1: A research _session_: +# by which we mean the activity of running a series of commands (interactively or via a script) that: +# - ingest some data, +# - manipulate it, and then +# - produce (and store) some outputs. +# +# ### 2: Types of commands: +# Whether interactive, or just running a final script, we can think of the commands that get run in a session as dividing into: +# - *manipulation* commands that load and transform data into the shape you want +# - *feedback* commands that report on your data - but are never intended to be exported. +# For example, running ```head()``` or ```describe()``` commands to make sure your manipulations have got the data into the format you want. +# - *query* commands that produce an output from your data (table/plot/regression model etc.) that you might want to export from the Trusted Research Environment (TRE) +# +# ### 3: Risk Assessment vs decision making: +# SACRO stands for Semi-Automated Checking of Research Outputs.
+# The prefix 'Semi' is important here - because in a principles-based system humans should make _decisions_ about output requests.
+# To help with that we provide the SACRO-Viewer, which collates all the relevant information for them. +# +# A key part of that information is the _Risk Assessment_. +# - Since it involves calculating metrics and comparing them to thresholds (the TRE's risk appetite) it can be done automatically, at the time an output query runs on the data. +# - This is what the ACRO tool does when you use it as part of your workflow. +# +# ### 4: What ACRO does +# The ACRO package aims to support you in producing *Safe Outputs* with minimal changes to your work flow. +# To do that we provide: +# - drop-in replacements for the most commonly used *output commands*, +# - keeping the same syntax as the originals, and +# - supporting as many of the options as we can (features supported will increase over time in response to demand). +# - a set of *session-management* commands to help you manage the set of files you request for output. +# - **Important to note** that currently acro outputs results (tables, details of regression models etc.) as `.csv` files.
+# - In other words we separate the processes of _creating_ outputs - which *must* be done *inside* the TRE.
+# from the process of _formatting_ them for publication - which can be done outside the TRE with your preferred toolchain. +# - ACRO handles creation. We are interested in hearing from researchers whether it is important to support them with formatting + +# %% [markdown] +# ## B: Getting Started with the demonstration +# +# ### Step 1: Setting up the environment with the tools we will use +# We will begin by importing some standard data science packages, and also the acro package itself. + +# %% +import os + +import numpy as np +import pandas as pd + +from acro import ACRO + +# %% [markdown] +# ### Step 2: Starting an ACRO session +# To do this we create an acro object by running the cell below. +# +# You can leave out the default parameters, but the cell below shows how you can: +# - provide the name of a *config* (risk appetite) file the TRE may have asked you to use +# - turn automatic suppression on or off right from the start of your session +# +# Note that when the cell runs it should report (in a different coloured font/background) +# - what version of acro is running: *this should be 0.4.12* +# - the TRE's risk appetite: that defines the rules your outputs will be checked against. +# - whether suppression is automatically applied to disclosive outputs. + +# %% +acro = ACRO(config="default", suppress=False) + +# %% [markdown] +# ### Step 3: Loading some test data +# +# The following cells in this step just contain standard *ingestion* and *manipulation* commands to load some data into a Pandas dataframe ready to be queried.
+# We will use some open-source data about nursery admissions. +# +# **There is no change to your workflow here** +# - Do whatever you want in this step! +# - We just assume you end up with your data in a pandas dataframe. +# +# + +# %% +from scipy.io.arff import loadarff + +##--- Manipulation commands --- +# specify where the data is +path = os.path.join("../data", "nursery.arff") + +# read it in using a common dataloader +data = loadarff(path) + + +# store in a pandas dataframe with some manipulation of type variable names +df = pd.DataFrame(data[0]) +df = df.select_dtypes([object]) +df = df.stack().str.decode("utf-8").unstack() +df.rename(columns={"class": "recommendation"}, inplace=True) + + +# make the children variable numeric +# so we can report statistics like mean etc. + +df["children"].replace(to_replace={"more": "4"}, inplace=True) +df["children"] = pd.to_numeric(df["children"]) + +df["children"] = df.apply( + lambda row: ( + row["children"] if row["children"] in (1, 2, 3) else np.random.randint(4, 10) + ), + axis=1, +) + +# %% +##--- Feedback Command ---- +# show the first 5 rows to make sure everything is how we would expect +df.head() + +# %% [markdown] +# ## C: Producing tables that are 'Safe Outputs' +# +# ### Producing tables using standard python libraries +# +# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. +# - There are hundreds (thousands?) of web sites showing how to do this. +# - You can make (hierarchical) 2-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row) +# - you can specify what the table cells contain by: +# - providing a statistic - for example: mean, count, std deviation, median etc.(pandas calls these *aggregation functions*) +# - specifying what variable to report on +# +# In the next example: +# - We will use the Pandas *crosstab()* function to create a summary table. +# +# - The table will calculate the **mean number of children** for every combination of **Recommendation** and **Parental status**. Going through the parameters in order: +# - `index=df.recommendation`: Defines the rows of the table using the recommendation column from the dataset. +# +# - `columns=df.parents`: Defines the columns of the table using the parents column from the dataset. +# - `values=df.children`: Specifies that we want to perform calculations on the children (the name of a column in the dataset). +# +# - `aggfunc="mean"`: Tells the function to calculate the **average (mean)** for those values. + +# %% +pd.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) + +# %% [markdown] +# ### Getting this table out of the TRE +# +# - If you submit just this table to get it out of the TRE, you're going to get an email from the output checkers asking you to produce the table of counts so that they can check minimum thresholds, and maybe some other data to help them check for 'dominance' if it is financial data. +# +# - Defining the disclosive cells by just looking at this table can be tricky. For example, did you spot that cell on the third row and third column with a mean of 1? How many families are being aggregated to get that suspiciously integer value? Would you say that this cell is safe or not safe, and why? +# +# - All this to-and-fro adds effort and delays to the process of getting your output from the TRE. +# +# - Acro is actually designed to help you with this! +# +# - Because you asked for a `mean` statistic, it knows about the possible risks and so checks them for you (and the output checkers). + +# %% [markdown] +# +# ### Producing tables using acro +# +# In the next example we will use **acro** to produce the same table. The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for. +# +# Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write +# ```` +# pd.crosstab(...) +# ```` +# so the only change is to use the prefix `acro.` rather than `pd.` +# +# _NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing. + +# %% +acro.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) + +# %% [markdown] +# ### How to understand this output +# The top part (with a pink background) is the result of the automatic risk analysis done in the background by acro. +# It is telling us that: +# - the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check +# - then it is showing which cells failed so you can choose how to respond +# - finally it is telling us that is has saved the table and risk assessment to our acro session with id "output_0" +# +# The part below is the normal output produced by the pandas _crosstab()_ function, same as the table we produced above using Pandas. +# +# +# ### How to respond to this input +# There are basically three choices: +# 1. We might decide these low numbers reveal something where the public interest outweighs the disclosure risk.
+# Rather than being a strict rules-based system, acro lets you attach an 'exception request' to an output, to send a message to the output checkers.
+# For example, you could type: +# ```` +# acro.add_exception('output_0',"I think you should let me have this because...") +# ```` +# +# 2. We redesign our data so that table so that none of the cells in the resulting table represent fewer than _n_ people (10 for the default risk appetite)
+# For example, we could recode _'very_recommend'_ and _'priority'_ into one label.
+# But maybe it is revealing that the _'recommend'_ label is not used? +# +# 3. We can redact the disclosive cells - and **acro will do this for us**.
+# We simply enable the option to suppress disclosive cells and re-run the query. +# +# The cell below shows option 3.b +# When you run the cell below you should see that: +# - the status now changes to `review` (so the output-checker knows what has been applied) +# - the code automatically adds an exception request saying that suppression has been applied +# - and, most importantly, the cells are redacted. + +# %% +acro.enable_suppression() +acro.crosstab( + index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" +) + +# %% [markdown] +# ## An example of a more complex table +# Just to show off the sort of tables that `crosstab()` can produce, let's make something more complex.
+# Going through the parameters in order: +# - passing a list of variable names to `index` (rather than a single variable/column name) tells it we want a hierarchy within the rows. +# - we can do the same to columns as well (or instead) if we want to +# - setting `values=df.children` tells it we want to report something about the number of children for each sub-group (table cell) +# - setting `aggfunc=mode` tells it the statistic we want to report is the mode of children +# - setting `margins=True` tells it to display row and column sub-totals +# +# It's worth noting that including the totals there are 6 columns in the risk assessment and 5 in the suppressed table.
+# This is because after suppression has replaced numbers with `NaN`, pandas removes the fully suppressed column (_'recommend'_) from the table. + +# %% +acro.suppress = True +acro.crosstab( + index=[df.parents, df.finance], + columns=df.recommendation, + values=df.children, + aggfunc="mode", + margins=True, +) + +# %% [markdown] +# ## D: What other sorts of analysis does ACRO currently support? +# We are continually adding support for more types of analysis as users prioritise them. +# +# ACRO currently supports: +# - **Tables** via `acro.crosstab()` and `acro.pivot_table()`. +# - supported aggregation functions are: _mean_, _median_, _sum_, _std_, _count_, and _mode_.
+# - **Survival analysis** via: `acro.surv_function()`, `acro.survival_table()` and `acro.survival_plot()`
+# - **Histograms** via:`acro.hist()`
+# - **Regression** via: `acro.ols()`, `acro.logit()`,`acro.probit()` +# with options for specifying formula in 'R-style' by adding the suffix 'r' e.g. `acro.olsr()` etc. +# +# You can get help on using any of these using the standard python `help()` syntax as shown in the next cell + +# %% +help(acro.logit) + +# %% [markdown] +# ## E: ACRO functionality to let users manage their outputs +# As explained above, you need to create an "acro session" whenever your code is run. +# +# After that, every time you run an acro `query' command both the output and the risk assessment are saved as part of the acro session. +# +# But we recognise that: +# - You may not want to request release of all your outputs - for example, the first table we produced above. +# - It is good practice to provide a more informative name than just *output_n* for the .csv files that acro produces +# - It helps the output checker if you provide some comments saying what the outputs are. +# - You might want to add more things to the bundles of files you want to take out, such as: +# - outputs from analyses that acro doesn't currently support +# - your code itself (which many journals want) +# - maybe a version of your paper in pdf/word format etc. +# +# Therefore acro provides the following commands for 'session management' +# ### 1: Listing the current contents of an ACRO session +# This output is not beautiful (there's a GUI coming soon) but it should let you identify outputs you want to rename,comment on, or delete. + +# %% +_ = acro.print_outputs() + +# %% [markdown] +# ### 2: Remove some ACRO outputs before finalising +# At the start of this demo we made a disclosive output -it;s the first one with status _fail_. +# +# We don't want to waste the output checker's time so lets remove it. + +# %% +acro.remove_output("output_0") + +# %% [markdown] +# ### 3: Rename ACRO outputs before finalising +# This is an example of renaming the outputs to provide more descriptive names. + +# %% +acro.rename_output("output_1", " crosstab_recommendation_vs_parents") +acro.rename_output("output_2", "mean_children_by_parents_finance_recommendation") + +# %% [markdown] +# ### 4: Add a comment to output +# This is an example of adding a comment to outputs. +# It can be used to provide a description or to pass additional information to the TRE staff.
+# They will see it alongside your file in the output checking viewer - rather than having it in an email somewhere. + +# %% +acro.add_comments( + "mean_children_by_parents_finance_recommendation", + "too few cases of recommend to report", +) + +# %% [markdown] +# ### 5. Request an exception +# An example of providing a reason why an exception should be made +# ```` +# acro.add_exception("output_n", "This is evidence of systematic bias?") +# ```` + +# %% [markdown] +# ### 6: Adding a custom output. +# +# As mentioned above you might want to request release of all sorts of things +# - including your code, +# - or outputs from analyses *acro* doesn't support (yet) +# +# In ACRO we can add a file to our session with a comment describing what it is + +# %% +acro.custom_output("acro_demo_2026.py", "This is the code that produced this session") + +# %% [markdown] +# ## F: Finishing your session and producing a folder of files to release. +# This is an example of the function _finalise()_ which the users must call at the end of each session. +# - It takes each output and saves it to a CSV file (or the original file type for custom outputs) +# - It also saves the SDC analysis for each output to a json file. +# - It adds checksums for everything - so we know they've not been edited. +# - It puts them all in a folder with the name you supply. +# +# **ACRO will not overwrite previous sessions** +# +# So every time you call finalise on a session you need to either: +# - delete the previous folder, or +# - provide a new folder name + +# %% +output = acro.finalise("my_acro_outputs_v1") + +# %% [markdown] +# ## G: Reminder about getting help while you work +# +# - if you remember the name of the command and want an explanation or to explain the syntax
+# from the python prompt type: ` help(acro.command_name)` +# +# +# - if you can't remember the name of the command, from the python prompt type: `help(acro.ACRO)` +# - not as user friendly but will list all the available commands + +# %% diff --git a/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt b/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt new file mode 100644 index 00000000..c15541d3 --- /dev/null +++ b/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt @@ -0,0 +1 @@ +ca915310f2df4af174d76e3ced68f3be2f8f06ec56a73bf7fa3218d81dd03b5e \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt b/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt new file mode 100644 index 00000000..fdf13d9f --- /dev/null +++ b/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt @@ -0,0 +1 @@ +5223236f9f7103586863e501f4c26ccf53fb693d82e32341b4bb039a76f9c97d \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt b/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt new file mode 100644 index 00000000..eb65627b --- /dev/null +++ b/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt @@ -0,0 +1 @@ +40242db45253cb1403aea412204e37fae903ce4bf3d2bbc184b7a50f6d9b833a \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/results.json.txt b/notebooks/my_acro_outputs_v1/checksums/results.json.txt new file mode 100644 index 00000000..85250b3b --- /dev/null +++ b/notebooks/my_acro_outputs_v1/checksums/results.json.txt @@ -0,0 +1 @@ +4071a8d79847888e075e3b7fb5e9d10a9b8dd722eb9e3c095edc2fa05e7c89a1 \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/config.json b/notebooks/my_acro_outputs_v1/config.json new file mode 100644 index 00000000..33108015 --- /dev/null +++ b/notebooks/my_acro_outputs_v1/config.json @@ -0,0 +1,10 @@ +{ + "safe_threshold": 10, + "safe_dof_threshold": 10, + "safe_nk_n": 2, + "safe_nk_k": 0.9, + "safe_pratio_p": 0.1, + "check_missing_values": false, + "survival_safe_threshold": 10, + "zeros_are_disclosive": true +} \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv b/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv new file mode 100644 index 00000000..2fa02fb5 --- /dev/null +++ b/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv @@ -0,0 +1,8 @@ +parents,finance,not_recom,priority,spec_prior,very_recom,All +great_pret,convenient,3.0,1.0,3.0,,1 +great_pret,inconv,2.0,1.0,3.0,,1 +pretentious,convenient,2.0,1.0,3.0,1.0,1 +pretentious,inconv,2.0,1.0,3.0,1.0,1 +usual,convenient,1.0,1.0,3.0,1.0,2 +usual,inconv,1.0,3.0,3.0,1.0,1 +All,,1.0,1.0,3.0,1.0,3 diff --git a/notebooks/my_acro_outputs_v1/results.json b/notebooks/my_acro_outputs_v1/results.json new file mode 100644 index 00000000..24ffc29e --- /dev/null +++ b/notebooks/my_acro_outputs_v1/results.json @@ -0,0 +1,256 @@ +{ + "version": "0.4.12", + "results": { + " crosstab_recommendation_vs_parents": { + "uid": " crosstab_recommendation_vs_parents", + "status": "review", + "type": "table", + "properties": { + "method": "crosstab" + }, + "files": [ + { + "name": " crosstab_recommendation_vs_parents_0.csv", + "sdc": { + "summary": { + "suppressed": true, + "negative": 0, + "missing": 0, + "threshold": 1, + "p-ratio": 4, + "nk-rule": 4, + "all-values-are-same": 0 + }, + "cells": { + "negative": [], + "missing": [], + "threshold": [ + [ + 2, + 2 + ] + ], + "p-ratio": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 4, + 0 + ] + ], + "nk-rule": [ + [ + 2, + 0 + ], + [ + 2, + 1 + ], + [ + 2, + 2 + ], + [ + 4, + 0 + ] + ], + "all-values-are-same": [] + } + } + } + ], + "outcome": { + "great_pret": { + "not_recom": "ok", + "priority": "ok", + "recommend": "p-ratio; nk-rule; ", + "spec_prior": "ok", + "very_recom": "p-ratio; nk-rule; " + }, + "pretentious": { + "not_recom": "ok", + "priority": "ok", + "recommend": "p-ratio; nk-rule; ", + "spec_prior": "ok", + "very_recom": "ok" + }, + "usual": { + "not_recom": "ok", + "priority": "ok", + "recommend": "threshold; p-ratio; nk-rule; ", + "spec_prior": "ok", + "very_recom": "ok" + } + }, + "command": "acro.crosstab(", + "summary": "review; threshold: 1 cells suppressed; p-ratio: 4 cells suppressed; nk-rule: 4 cells suppressed; ", + "timestamp": "2026-06-30T15:19:49.603861", + "comments": [], + "exception": "Suppression automatically applied where needed" + }, + "mean_children_by_parents_finance_recommendation": { + "uid": "mean_children_by_parents_finance_recommendation", + "status": "review", + "type": "table", + "properties": { + "method": "crosstab" + }, + "files": [ + { + "name": "mean_children_by_parents_finance_recommendation_0.csv", + "sdc": { + "summary": { + "suppressed": true, + "negative": 0, + "missing": 0, + "threshold": 0, + "p-ratio": 0, + "nk-rule": 0, + "all-values-are-same": 9 + }, + "cells": { + "negative": [], + "missing": [], + "threshold": [], + "p-ratio": [], + "nk-rule": [], + "all-values-are-same": [ + [ + 0, + 2 + ], + [ + 0, + 4 + ], + [ + 1, + 2 + ], + [ + 1, + 4 + ], + [ + 2, + 2 + ], + [ + 3, + 2 + ], + [ + 4, + 2 + ], + [ + 5, + 2 + ], + [ + 6, + 2 + ] + ] + } + } + } + ], + "outcome": { + "not_recom": { + "('great_pret', 'convenient')": "ok", + "('great_pret', 'inconv')": "ok", + "('pretentious', 'convenient')": "ok", + "('pretentious', 'inconv')": "ok", + "('usual', 'convenient')": "ok", + "('usual', 'inconv')": "ok", + "('All', '')": "ok" + }, + "priority": { + "('great_pret', 'convenient')": "ok", + "('great_pret', 'inconv')": "ok", + "('pretentious', 'convenient')": "ok", + "('pretentious', 'inconv')": "ok", + "('usual', 'convenient')": "ok", + "('usual', 'inconv')": "ok", + "('All', '')": "ok" + }, + "recommend": { + "('great_pret', 'convenient')": "all-values-are-same; ", + "('great_pret', 'inconv')": "all-values-are-same; ", + "('pretentious', 'convenient')": "all-values-are-same; ", + "('pretentious', 'inconv')": "all-values-are-same; ", + "('usual', 'convenient')": "all-values-are-same; ", + "('usual', 'inconv')": "all-values-are-same; ", + "('All', '')": "all-values-are-same; " + }, + "spec_prior": { + "('great_pret', 'convenient')": "ok", + "('great_pret', 'inconv')": "ok", + "('pretentious', 'convenient')": "ok", + "('pretentious', 'inconv')": "ok", + "('usual', 'convenient')": "ok", + "('usual', 'inconv')": "ok", + "('All', '')": "ok" + }, + "very_recom": { + "('great_pret', 'convenient')": "all-values-are-same; ", + "('great_pret', 'inconv')": "all-values-are-same; ", + "('pretentious', 'convenient')": "ok", + "('pretentious', 'inconv')": "ok", + "('usual', 'convenient')": "ok", + "('usual', 'inconv')": "ok", + "('All', '')": "ok" + }, + "All": { + "('great_pret', 'convenient')": "ok", + "('great_pret', 'inconv')": "ok", + "('pretentious', 'convenient')": "ok", + "('pretentious', 'inconv')": "ok", + "('usual', 'convenient')": "ok", + "('usual', 'inconv')": "ok", + "('All', '')": "ok" + } + }, + "command": "acro.crosstab(", + "summary": "review; all-values-are-same: 9 cells suppressed; ", + "timestamp": "2026-06-30T15:19:49.815485", + "comments": [ + "too few cases of recommend to report" + ], + "exception": "Suppression automatically applied where needed" + }, + "output_3": { + "uid": "output_3", + "status": "review", + "type": "custom", + "properties": {}, + "files": [ + { + "name": "acro_demo_2026.py", + "sdc": {} + } + ], + "outcome": {}, + "command": "custom", + "summary": "review", + "timestamp": "2026-06-30T15:19:49.906273", + "comments": [ + "This is the code that produced this session" + ], + "exception": "" + } + } +} \ No newline at end of file From 7851ce3b29030180725411987b6676754d71572d Mon Sep 17 00:00:00 2001 From: mahaalbashir Date: Tue, 30 Jun 2026 15:20:52 +0100 Subject: [PATCH 6/6] updating notebooks --- .../ crosstab_recommendation_vs_parents_0.csv | 6 - .../my_acro_outputs_v1/acro_demo_2026.py | 372 ------------------ ...osstab_recommendation_vs_parents_0.csv.txt | 1 - .../checksums/acro_demo_2026.py.txt | 1 - ...y_parents_finance_recommendation_0.csv.txt | 1 - .../checksums/results.json.txt | 1 - notebooks/my_acro_outputs_v1/config.json | 10 - ...en_by_parents_finance_recommendation_0.csv | 8 - notebooks/my_acro_outputs_v1/results.json | 256 ------------ 9 files changed, 656 deletions(-) delete mode 100644 notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv delete mode 100644 notebooks/my_acro_outputs_v1/acro_demo_2026.py delete mode 100644 notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt delete mode 100644 notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt delete mode 100644 notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt delete mode 100644 notebooks/my_acro_outputs_v1/checksums/results.json.txt delete mode 100644 notebooks/my_acro_outputs_v1/config.json delete mode 100644 notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv delete mode 100644 notebooks/my_acro_outputs_v1/results.json diff --git a/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv b/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv deleted file mode 100644 index 07ed46fa..00000000 --- a/notebooks/my_acro_outputs_v1/ crosstab_recommendation_vs_parents_0.csv +++ /dev/null @@ -1,6 +0,0 @@ -recommendation,great_pret,pretentious,usual -not_recom,3.1006944444444446,3.1326388888888888,3.1305555555555555 -priority,2.625874125874126,3.039757412398922,3.1237006237006235 -recommend,,, -spec_prior,3.3471810089020773,3.3757911392405062,3.401055408970976 -very_recom,,2.0984848484848486,2.25 diff --git a/notebooks/my_acro_outputs_v1/acro_demo_2026.py b/notebooks/my_acro_outputs_v1/acro_demo_2026.py deleted file mode 100644 index 5b62e94b..00000000 --- a/notebooks/my_acro_outputs_v1/acro_demo_2026.py +++ /dev/null @@ -1,372 +0,0 @@ -""" -Acro_demo_2026.py. - -Created (except this docstring) from acro_demo_2026.iynb -author Jim Smith. -""" - -# # ACRO Demonstration -# This is a simple notebook to get you started with using the ```acro``` package to add disclosure risk control to your analysis. -# -# ### Assumptions -# For the purpose of this tutorial we assume some minimal prior experience with using python for data science. -# In particular the use of the industry-standard Pandas package for: -# - storing and manipulating datasets -# - creating basic tables, pivot_tables, and plots (e.g. histograms) -# -# This example is a Jupyter notebook split into cells. -# - Cells may contain code or text/images, and normally they are processed by stepping through them one-by-one. -# - To run (or render) a cell click the *run* icon (at the top of the page) or *shift-return* on your keyboard. -# That will display any output created (for a code cell) and move the focus to the next cell. - -# %% [markdown] -# ## A: The basic concepts -# ### 1: A research _session_: -# by which we mean the activity of running a series of commands (interactively or via a script) that: -# - ingest some data, -# - manipulate it, and then -# - produce (and store) some outputs. -# -# ### 2: Types of commands: -# Whether interactive, or just running a final script, we can think of the commands that get run in a session as dividing into: -# - *manipulation* commands that load and transform data into the shape you want -# - *feedback* commands that report on your data - but are never intended to be exported. -# For example, running ```head()``` or ```describe()``` commands to make sure your manipulations have got the data into the format you want. -# - *query* commands that produce an output from your data (table/plot/regression model etc.) that you might want to export from the Trusted Research Environment (TRE) -# -# ### 3: Risk Assessment vs decision making: -# SACRO stands for Semi-Automated Checking of Research Outputs.
-# The prefix 'Semi' is important here - because in a principles-based system humans should make _decisions_ about output requests.
-# To help with that we provide the SACRO-Viewer, which collates all the relevant information for them. -# -# A key part of that information is the _Risk Assessment_. -# - Since it involves calculating metrics and comparing them to thresholds (the TRE's risk appetite) it can be done automatically, at the time an output query runs on the data. -# - This is what the ACRO tool does when you use it as part of your workflow. -# -# ### 4: What ACRO does -# The ACRO package aims to support you in producing *Safe Outputs* with minimal changes to your work flow. -# To do that we provide: -# - drop-in replacements for the most commonly used *output commands*, -# - keeping the same syntax as the originals, and -# - supporting as many of the options as we can (features supported will increase over time in response to demand). -# - a set of *session-management* commands to help you manage the set of files you request for output. -# - **Important to note** that currently acro outputs results (tables, details of regression models etc.) as `.csv` files.
-# - In other words we separate the processes of _creating_ outputs - which *must* be done *inside* the TRE.
-# from the process of _formatting_ them for publication - which can be done outside the TRE with your preferred toolchain. -# - ACRO handles creation. We are interested in hearing from researchers whether it is important to support them with formatting - -# %% [markdown] -# ## B: Getting Started with the demonstration -# -# ### Step 1: Setting up the environment with the tools we will use -# We will begin by importing some standard data science packages, and also the acro package itself. - -# %% -import os - -import numpy as np -import pandas as pd - -from acro import ACRO - -# %% [markdown] -# ### Step 2: Starting an ACRO session -# To do this we create an acro object by running the cell below. -# -# You can leave out the default parameters, but the cell below shows how you can: -# - provide the name of a *config* (risk appetite) file the TRE may have asked you to use -# - turn automatic suppression on or off right from the start of your session -# -# Note that when the cell runs it should report (in a different coloured font/background) -# - what version of acro is running: *this should be 0.4.12* -# - the TRE's risk appetite: that defines the rules your outputs will be checked against. -# - whether suppression is automatically applied to disclosive outputs. - -# %% -acro = ACRO(config="default", suppress=False) - -# %% [markdown] -# ### Step 3: Loading some test data -# -# The following cells in this step just contain standard *ingestion* and *manipulation* commands to load some data into a Pandas dataframe ready to be queried.
-# We will use some open-source data about nursery admissions. -# -# **There is no change to your workflow here** -# - Do whatever you want in this step! -# - We just assume you end up with your data in a pandas dataframe. -# -# - -# %% -from scipy.io.arff import loadarff - -##--- Manipulation commands --- -# specify where the data is -path = os.path.join("../data", "nursery.arff") - -# read it in using a common dataloader -data = loadarff(path) - - -# store in a pandas dataframe with some manipulation of type variable names -df = pd.DataFrame(data[0]) -df = df.select_dtypes([object]) -df = df.stack().str.decode("utf-8").unstack() -df.rename(columns={"class": "recommendation"}, inplace=True) - - -# make the children variable numeric -# so we can report statistics like mean etc. - -df["children"].replace(to_replace={"more": "4"}, inplace=True) -df["children"] = pd.to_numeric(df["children"]) - -df["children"] = df.apply( - lambda row: ( - row["children"] if row["children"] in (1, 2, 3) else np.random.randint(4, 10) - ), - axis=1, -) - -# %% -##--- Feedback Command ---- -# show the first 5 rows to make sure everything is how we would expect -df.head() - -# %% [markdown] -# ## C: Producing tables that are 'Safe Outputs' -# -# ### Producing tables using standard python libraries -# -# The easiest way to make tables in python is to use the industry-standard pandas *crosstab()* function. -# - There are hundreds (thousands?) of web sites showing how to do this. -# - You can make (hierarchical) 2-D tables (or 1-D if you add a 'dummy' variable containing the same value for each row) -# - you can specify what the table cells contain by: -# - providing a statistic - for example: mean, count, std deviation, median etc.(pandas calls these *aggregation functions*) -# - specifying what variable to report on -# -# In the next example: -# - We will use the Pandas *crosstab()* function to create a summary table. -# -# - The table will calculate the **mean number of children** for every combination of **Recommendation** and **Parental status**. Going through the parameters in order: -# - `index=df.recommendation`: Defines the rows of the table using the recommendation column from the dataset. -# -# - `columns=df.parents`: Defines the columns of the table using the parents column from the dataset. -# - `values=df.children`: Specifies that we want to perform calculations on the children (the name of a column in the dataset). -# -# - `aggfunc="mean"`: Tells the function to calculate the **average (mean)** for those values. - -# %% -pd.crosstab( - index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" -) - -# %% [markdown] -# ### Getting this table out of the TRE -# -# - If you submit just this table to get it out of the TRE, you're going to get an email from the output checkers asking you to produce the table of counts so that they can check minimum thresholds, and maybe some other data to help them check for 'dominance' if it is financial data. -# -# - Defining the disclosive cells by just looking at this table can be tricky. For example, did you spot that cell on the third row and third column with a mean of 1? How many families are being aggregated to get that suspiciously integer value? Would you say that this cell is safe or not safe, and why? -# -# - All this to-and-fro adds effort and delays to the process of getting your output from the TRE. -# -# - Acro is actually designed to help you with this! -# -# - Because you asked for a `mean` statistic, it knows about the possible risks and so checks them for you (and the output checkers). - -# %% [markdown] -# -# ### Producing tables using acro -# -# In the next example we will use **acro** to produce the same table. The acro version uses all the pandas code - but it adds extra code that checks for disclosure risks depending on the statistic you ask for. -# -# Note that having imported the pandas package with the shortname `pd`(most people do) you would normally write -# ```` -# pd.crosstab(...) -# ```` -# so the only change is to use the prefix `acro.` rather than `pd.` -# -# _NB_: the first two parameters to crosstab() are mandatory, so you could just do `crosstab(df.recommendation,df.parents)` to save typing. - -# %% -acro.crosstab( - index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" -) - -# %% [markdown] -# ### How to understand this output -# The top part (with a pink background) is the result of the automatic risk analysis done in the background by acro. -# It is telling us that: -# - the overall summary is _fail_ because 4 cells are failing the 'p-ratio and nk-rule' checks and 1 cell is failing the 'minimum threshold' check -# - then it is showing which cells failed so you can choose how to respond -# - finally it is telling us that is has saved the table and risk assessment to our acro session with id "output_0" -# -# The part below is the normal output produced by the pandas _crosstab()_ function, same as the table we produced above using Pandas. -# -# -# ### How to respond to this input -# There are basically three choices: -# 1. We might decide these low numbers reveal something where the public interest outweighs the disclosure risk.
-# Rather than being a strict rules-based system, acro lets you attach an 'exception request' to an output, to send a message to the output checkers.
-# For example, you could type: -# ```` -# acro.add_exception('output_0',"I think you should let me have this because...") -# ```` -# -# 2. We redesign our data so that table so that none of the cells in the resulting table represent fewer than _n_ people (10 for the default risk appetite)
-# For example, we could recode _'very_recommend'_ and _'priority'_ into one label.
-# But maybe it is revealing that the _'recommend'_ label is not used? -# -# 3. We can redact the disclosive cells - and **acro will do this for us**.
-# We simply enable the option to suppress disclosive cells and re-run the query. -# -# The cell below shows option 3.b -# When you run the cell below you should see that: -# - the status now changes to `review` (so the output-checker knows what has been applied) -# - the code automatically adds an exception request saying that suppression has been applied -# - and, most importantly, the cells are redacted. - -# %% -acro.enable_suppression() -acro.crosstab( - index=df.recommendation, columns=df.parents, values=df.children, aggfunc="mean" -) - -# %% [markdown] -# ## An example of a more complex table -# Just to show off the sort of tables that `crosstab()` can produce, let's make something more complex.
-# Going through the parameters in order: -# - passing a list of variable names to `index` (rather than a single variable/column name) tells it we want a hierarchy within the rows. -# - we can do the same to columns as well (or instead) if we want to -# - setting `values=df.children` tells it we want to report something about the number of children for each sub-group (table cell) -# - setting `aggfunc=mode` tells it the statistic we want to report is the mode of children -# - setting `margins=True` tells it to display row and column sub-totals -# -# It's worth noting that including the totals there are 6 columns in the risk assessment and 5 in the suppressed table.
-# This is because after suppression has replaced numbers with `NaN`, pandas removes the fully suppressed column (_'recommend'_) from the table. - -# %% -acro.suppress = True -acro.crosstab( - index=[df.parents, df.finance], - columns=df.recommendation, - values=df.children, - aggfunc="mode", - margins=True, -) - -# %% [markdown] -# ## D: What other sorts of analysis does ACRO currently support? -# We are continually adding support for more types of analysis as users prioritise them. -# -# ACRO currently supports: -# - **Tables** via `acro.crosstab()` and `acro.pivot_table()`. -# - supported aggregation functions are: _mean_, _median_, _sum_, _std_, _count_, and _mode_.
-# - **Survival analysis** via: `acro.surv_function()`, `acro.survival_table()` and `acro.survival_plot()`
-# - **Histograms** via:`acro.hist()`
-# - **Regression** via: `acro.ols()`, `acro.logit()`,`acro.probit()` -# with options for specifying formula in 'R-style' by adding the suffix 'r' e.g. `acro.olsr()` etc. -# -# You can get help on using any of these using the standard python `help()` syntax as shown in the next cell - -# %% -help(acro.logit) - -# %% [markdown] -# ## E: ACRO functionality to let users manage their outputs -# As explained above, you need to create an "acro session" whenever your code is run. -# -# After that, every time you run an acro `query' command both the output and the risk assessment are saved as part of the acro session. -# -# But we recognise that: -# - You may not want to request release of all your outputs - for example, the first table we produced above. -# - It is good practice to provide a more informative name than just *output_n* for the .csv files that acro produces -# - It helps the output checker if you provide some comments saying what the outputs are. -# - You might want to add more things to the bundles of files you want to take out, such as: -# - outputs from analyses that acro doesn't currently support -# - your code itself (which many journals want) -# - maybe a version of your paper in pdf/word format etc. -# -# Therefore acro provides the following commands for 'session management' -# ### 1: Listing the current contents of an ACRO session -# This output is not beautiful (there's a GUI coming soon) but it should let you identify outputs you want to rename,comment on, or delete. - -# %% -_ = acro.print_outputs() - -# %% [markdown] -# ### 2: Remove some ACRO outputs before finalising -# At the start of this demo we made a disclosive output -it;s the first one with status _fail_. -# -# We don't want to waste the output checker's time so lets remove it. - -# %% -acro.remove_output("output_0") - -# %% [markdown] -# ### 3: Rename ACRO outputs before finalising -# This is an example of renaming the outputs to provide more descriptive names. - -# %% -acro.rename_output("output_1", " crosstab_recommendation_vs_parents") -acro.rename_output("output_2", "mean_children_by_parents_finance_recommendation") - -# %% [markdown] -# ### 4: Add a comment to output -# This is an example of adding a comment to outputs. -# It can be used to provide a description or to pass additional information to the TRE staff.
-# They will see it alongside your file in the output checking viewer - rather than having it in an email somewhere. - -# %% -acro.add_comments( - "mean_children_by_parents_finance_recommendation", - "too few cases of recommend to report", -) - -# %% [markdown] -# ### 5. Request an exception -# An example of providing a reason why an exception should be made -# ```` -# acro.add_exception("output_n", "This is evidence of systematic bias?") -# ```` - -# %% [markdown] -# ### 6: Adding a custom output. -# -# As mentioned above you might want to request release of all sorts of things -# - including your code, -# - or outputs from analyses *acro* doesn't support (yet) -# -# In ACRO we can add a file to our session with a comment describing what it is - -# %% -acro.custom_output("acro_demo_2026.py", "This is the code that produced this session") - -# %% [markdown] -# ## F: Finishing your session and producing a folder of files to release. -# This is an example of the function _finalise()_ which the users must call at the end of each session. -# - It takes each output and saves it to a CSV file (or the original file type for custom outputs) -# - It also saves the SDC analysis for each output to a json file. -# - It adds checksums for everything - so we know they've not been edited. -# - It puts them all in a folder with the name you supply. -# -# **ACRO will not overwrite previous sessions** -# -# So every time you call finalise on a session you need to either: -# - delete the previous folder, or -# - provide a new folder name - -# %% -output = acro.finalise("my_acro_outputs_v1") - -# %% [markdown] -# ## G: Reminder about getting help while you work -# -# - if you remember the name of the command and want an explanation or to explain the syntax
-# from the python prompt type: ` help(acro.command_name)` -# -# -# - if you can't remember the name of the command, from the python prompt type: `help(acro.ACRO)` -# - not as user friendly but will list all the available commands - -# %% diff --git a/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt b/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt deleted file mode 100644 index c15541d3..00000000 --- a/notebooks/my_acro_outputs_v1/checksums/ crosstab_recommendation_vs_parents_0.csv.txt +++ /dev/null @@ -1 +0,0 @@ -ca915310f2df4af174d76e3ced68f3be2f8f06ec56a73bf7fa3218d81dd03b5e \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt b/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt deleted file mode 100644 index fdf13d9f..00000000 --- a/notebooks/my_acro_outputs_v1/checksums/acro_demo_2026.py.txt +++ /dev/null @@ -1 +0,0 @@ -5223236f9f7103586863e501f4c26ccf53fb693d82e32341b4bb039a76f9c97d \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt b/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt deleted file mode 100644 index eb65627b..00000000 --- a/notebooks/my_acro_outputs_v1/checksums/mean_children_by_parents_finance_recommendation_0.csv.txt +++ /dev/null @@ -1 +0,0 @@ -40242db45253cb1403aea412204e37fae903ce4bf3d2bbc184b7a50f6d9b833a \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/checksums/results.json.txt b/notebooks/my_acro_outputs_v1/checksums/results.json.txt deleted file mode 100644 index 85250b3b..00000000 --- a/notebooks/my_acro_outputs_v1/checksums/results.json.txt +++ /dev/null @@ -1 +0,0 @@ -4071a8d79847888e075e3b7fb5e9d10a9b8dd722eb9e3c095edc2fa05e7c89a1 \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/config.json b/notebooks/my_acro_outputs_v1/config.json deleted file mode 100644 index 33108015..00000000 --- a/notebooks/my_acro_outputs_v1/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "safe_threshold": 10, - "safe_dof_threshold": 10, - "safe_nk_n": 2, - "safe_nk_k": 0.9, - "safe_pratio_p": 0.1, - "check_missing_values": false, - "survival_safe_threshold": 10, - "zeros_are_disclosive": true -} \ No newline at end of file diff --git a/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv b/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv deleted file mode 100644 index 2fa02fb5..00000000 --- a/notebooks/my_acro_outputs_v1/mean_children_by_parents_finance_recommendation_0.csv +++ /dev/null @@ -1,8 +0,0 @@ -parents,finance,not_recom,priority,spec_prior,very_recom,All -great_pret,convenient,3.0,1.0,3.0,,1 -great_pret,inconv,2.0,1.0,3.0,,1 -pretentious,convenient,2.0,1.0,3.0,1.0,1 -pretentious,inconv,2.0,1.0,3.0,1.0,1 -usual,convenient,1.0,1.0,3.0,1.0,2 -usual,inconv,1.0,3.0,3.0,1.0,1 -All,,1.0,1.0,3.0,1.0,3 diff --git a/notebooks/my_acro_outputs_v1/results.json b/notebooks/my_acro_outputs_v1/results.json deleted file mode 100644 index 24ffc29e..00000000 --- a/notebooks/my_acro_outputs_v1/results.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "version": "0.4.12", - "results": { - " crosstab_recommendation_vs_parents": { - "uid": " crosstab_recommendation_vs_parents", - "status": "review", - "type": "table", - "properties": { - "method": "crosstab" - }, - "files": [ - { - "name": " crosstab_recommendation_vs_parents_0.csv", - "sdc": { - "summary": { - "suppressed": true, - "negative": 0, - "missing": 0, - "threshold": 1, - "p-ratio": 4, - "nk-rule": 4, - "all-values-are-same": 0 - }, - "cells": { - "negative": [], - "missing": [], - "threshold": [ - [ - 2, - 2 - ] - ], - "p-ratio": [ - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 4, - 0 - ] - ], - "nk-rule": [ - [ - 2, - 0 - ], - [ - 2, - 1 - ], - [ - 2, - 2 - ], - [ - 4, - 0 - ] - ], - "all-values-are-same": [] - } - } - } - ], - "outcome": { - "great_pret": { - "not_recom": "ok", - "priority": "ok", - "recommend": "p-ratio; nk-rule; ", - "spec_prior": "ok", - "very_recom": "p-ratio; nk-rule; " - }, - "pretentious": { - "not_recom": "ok", - "priority": "ok", - "recommend": "p-ratio; nk-rule; ", - "spec_prior": "ok", - "very_recom": "ok" - }, - "usual": { - "not_recom": "ok", - "priority": "ok", - "recommend": "threshold; p-ratio; nk-rule; ", - "spec_prior": "ok", - "very_recom": "ok" - } - }, - "command": "acro.crosstab(", - "summary": "review; threshold: 1 cells suppressed; p-ratio: 4 cells suppressed; nk-rule: 4 cells suppressed; ", - "timestamp": "2026-06-30T15:19:49.603861", - "comments": [], - "exception": "Suppression automatically applied where needed" - }, - "mean_children_by_parents_finance_recommendation": { - "uid": "mean_children_by_parents_finance_recommendation", - "status": "review", - "type": "table", - "properties": { - "method": "crosstab" - }, - "files": [ - { - "name": "mean_children_by_parents_finance_recommendation_0.csv", - "sdc": { - "summary": { - "suppressed": true, - "negative": 0, - "missing": 0, - "threshold": 0, - "p-ratio": 0, - "nk-rule": 0, - "all-values-are-same": 9 - }, - "cells": { - "negative": [], - "missing": [], - "threshold": [], - "p-ratio": [], - "nk-rule": [], - "all-values-are-same": [ - [ - 0, - 2 - ], - [ - 0, - 4 - ], - [ - 1, - 2 - ], - [ - 1, - 4 - ], - [ - 2, - 2 - ], - [ - 3, - 2 - ], - [ - 4, - 2 - ], - [ - 5, - 2 - ], - [ - 6, - 2 - ] - ] - } - } - } - ], - "outcome": { - "not_recom": { - "('great_pret', 'convenient')": "ok", - "('great_pret', 'inconv')": "ok", - "('pretentious', 'convenient')": "ok", - "('pretentious', 'inconv')": "ok", - "('usual', 'convenient')": "ok", - "('usual', 'inconv')": "ok", - "('All', '')": "ok" - }, - "priority": { - "('great_pret', 'convenient')": "ok", - "('great_pret', 'inconv')": "ok", - "('pretentious', 'convenient')": "ok", - "('pretentious', 'inconv')": "ok", - "('usual', 'convenient')": "ok", - "('usual', 'inconv')": "ok", - "('All', '')": "ok" - }, - "recommend": { - "('great_pret', 'convenient')": "all-values-are-same; ", - "('great_pret', 'inconv')": "all-values-are-same; ", - "('pretentious', 'convenient')": "all-values-are-same; ", - "('pretentious', 'inconv')": "all-values-are-same; ", - "('usual', 'convenient')": "all-values-are-same; ", - "('usual', 'inconv')": "all-values-are-same; ", - "('All', '')": "all-values-are-same; " - }, - "spec_prior": { - "('great_pret', 'convenient')": "ok", - "('great_pret', 'inconv')": "ok", - "('pretentious', 'convenient')": "ok", - "('pretentious', 'inconv')": "ok", - "('usual', 'convenient')": "ok", - "('usual', 'inconv')": "ok", - "('All', '')": "ok" - }, - "very_recom": { - "('great_pret', 'convenient')": "all-values-are-same; ", - "('great_pret', 'inconv')": "all-values-are-same; ", - "('pretentious', 'convenient')": "ok", - "('pretentious', 'inconv')": "ok", - "('usual', 'convenient')": "ok", - "('usual', 'inconv')": "ok", - "('All', '')": "ok" - }, - "All": { - "('great_pret', 'convenient')": "ok", - "('great_pret', 'inconv')": "ok", - "('pretentious', 'convenient')": "ok", - "('pretentious', 'inconv')": "ok", - "('usual', 'convenient')": "ok", - "('usual', 'inconv')": "ok", - "('All', '')": "ok" - } - }, - "command": "acro.crosstab(", - "summary": "review; all-values-are-same: 9 cells suppressed; ", - "timestamp": "2026-06-30T15:19:49.815485", - "comments": [ - "too few cases of recommend to report" - ], - "exception": "Suppression automatically applied where needed" - }, - "output_3": { - "uid": "output_3", - "status": "review", - "type": "custom", - "properties": {}, - "files": [ - { - "name": "acro_demo_2026.py", - "sdc": {} - } - ], - "outcome": {}, - "command": "custom", - "summary": "review", - "timestamp": "2026-06-30T15:19:49.906273", - "comments": [ - "This is the code that produced this session" - ], - "exception": "" - } - } -} \ No newline at end of file