From 577c421a0c70780a1c2fd0c6f2868957bccb5a36 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 14:41:22 +0100 Subject: [PATCH 1/8] Midway pause --- .../configure_recipe/bin/configure_recipe.py | 144 ++---------------- .../etc/esmvaltool_config.yml | 51 +++++++ CMEW/opt/rose-suite-metoffice.conf | 5 +- 3 files changed, 63 insertions(+), 137 deletions(-) create mode 100644 CMEW/app/configure_recipe/etc/esmvaltool_config.yml diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index 8c2f4765..e3007014 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -20,18 +20,16 @@ def main(): Write the required user and developer configuration files for ESMValTool. """ + # Retrieve relevant environment variables values = retrieve_values_from_task_env() logger.info("Retrieving values") - developer_config_path = values["DEV_CONFIG_PATH"] - logger.info("Creating developer config") - developer_config_contents = create_developer_config(values) - ensure_parent_dir(developer_config_path) - logger.info("Writing developer config to %s", developer_config_path) - write_yaml(developer_config_path, developer_config_contents) + # Create a single configuration file user_config_path = values["USER_CONFIG_PATH"] logger.info("Creating user config") user_config_contents = create_user_config(values) + + # Write the file out ensure_parent_dir(user_config_path) logger.info("Writing user config to %s", user_config_path) write_yaml(user_config_path, user_config_contents) @@ -51,12 +49,11 @@ def retrieve_values_from_task_env(): values_from_task_env = { "CYLC_WORKFLOW_SHARE_DIR": os.environ["CYLC_WORKFLOW_SHARE_DIR"], "DEV_CONFIG_PATH": os.environ["DEV_CONFIG_PATH"], - "DRS_CMIP6": os.environ["DRS_CMIP6"], - "DRS_OBS4MIPS": os.environ["DRS_OBS4MIPS"], "MAX_PARALLEL_TASKS": os.environ["MAX_PARALLEL_TASKS"], "MIP_TABLE_DIR": os.environ["MIP_TABLE_DIR"], "OUTPUT_DIR": os.environ["OUTPUT_DIR"], "ROOTPATH_CMIP6": os.environ["ROOTPATH_CMIP6"], + "ROOTPATH_OBS": os.environ["ROOTPATH_OBS"], "ROOTPATH_OBS4MIPS": os.environ["ROOTPATH_OBS4MIPS"], "USER_CONFIG_PATH": os.environ["USER_CONFIG_PATH"], } @@ -64,115 +61,8 @@ def retrieve_values_from_task_env(): return values_from_task_env -def create_developer_config(values): - """ - Return the contents of the developer configuration file. +def retrieve_default_values(): - Parameters - ---------- - values : dict - Configuration values. - - Returns - ------- - dict - Developer configuration content. - """ - mip_table_dir = values["MIP_TABLE_DIR"] - - developer_config_file_contents = { - "custom": { - "cmor_path": mip_table_dir, - }, - "ESMVal": { - "cmor_strict": True, - "input_dir": { - "default": "/", - "BADC": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "DKRZ": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "ESGF": ( - "{project}/{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "ETHZ": ( - "{exp}/{mip}/{short_name}/{dataset}/" "{ensemble}/{grid}/" - ), - "SYNDA": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - }, - "input_file": ( - "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" - ), - "output_file": ( - "{project}_{dataset}_{mip}_{exp}_{ensemble}_" - "{short_name}_{grid}" - ), - "cmor_type": "CMIP6", - "cmor_default_table_prefix": "GCModelDev_", - }, - "CMIP6": { - "cmor_strict": True, - "input_dir": { - "default": "/", - "BADC": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "DKRZ": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "ESGF": ( - "{project}/{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - "ETHZ": ( - "{exp}/{mip}/{short_name}/{dataset}/" "{ensemble}/{grid}/" - ), - "SYNDA": ( - "{activity}/{institute}/{dataset}/{exp}/" - "{ensemble}/{mip}/{short_name}/{grid}/{version}" - ), - }, - "input_file": ( - "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" - ), - "output_file": ( - "{project}_{dataset}_{mip}_{exp}_{ensemble}_" - "{short_name}_{grid}" - ), - "cmor_type": "CMIP6", - }, - "obs4MIPs": { - "cmor_strict": False, - "input_dir": { - "default": "Tier{tier}/{dataset}", - "ESGF": "{project}/{dataset}/{version}", - "RCAST": "/", - "IPSL": ( - "{realm}/{short_name}/{freq}/{grid}/" - "{institute}/{dataset}/{latest_version}" - ), - }, - "input_file": { - "default": "{short_name}_*.nc", - "ESGF": "{short_name}_*.nc", - }, - "output_file": "{project}_{dataset}_{short_name}", - "cmor_type": "CMIP6", - "cmor_path": "obs4mips", - "cmor_default_table_prefix": "obs4MIPs_", - }, - } - return developer_config_file_contents def create_user_config(values=None): @@ -200,11 +90,6 @@ def create_user_config(values=None): ) logger.debug("esmval: %s", esmval) - if "MAX_PARALLEL_TASKS" in values: - max_parallel_tasks = int(values["MAX_PARALLEL_TASKS"]) - else: - max_parallel_tasks = None - # Note that 'auxiliary_data_dir' and 'download_dir' # are set to empty values and cannot currently be # configured. However, 'download_dir' is used only when using the @@ -214,23 +99,14 @@ def create_user_config(values=None): # additional datasets, so may need to be configured in the future. user_config_file_contents = { - "auxiliary_data_dir": "", - "config_developer_file": values.get("DEV_CONFIG_PATH"), - "download_dir": "", - "drs": { - "CMIP6": values.get("DRS_CMIP6"), - "obs4MIPs": values.get("DRS_OBS4MIPS"), - "ESMVal": "BADC", - }, "max_parallel_tasks": max_parallel_tasks, "output_dir": values.get("OUTPUT_DIR"), - "remove_preproc_dir": False, - "rootpath": { + "CMIP6": values.get("ROOTPATH_CMIP6"), - "obs4MIPs": values.get("ROOTPATH_OBS4MIPS"), + "OBS": values.get("ROOTPATH_OBS"), + "OBS4MIPs": values.get("ROOTPATH_OBS4MIPS"), "ESMVal": esmval, - }, - } + logger.debug("User config file contents:\n%s", user_config_file_contents) return user_config_file_contents diff --git a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml new file mode 100644 index 00000000..36760c4e --- /dev/null +++ b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml @@ -0,0 +1,51 @@ +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. + +--- +# Destination directory where all output will be written +output_dir: /this/is/overwritten + +# Data filepath structures +projects: + ESMVal: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: /this/is/overwritten + dirname_template: "{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" + CMIP6: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: /this/is/overwritten + dirname_template: "{activity}/{institute}/{dataset}/{exp}/{ensemble}/{mip}/{short_name}/{grid}/{version}" + filename_template: "{short_name}_{mip}_{dataset}_{exp}_{ensemble}_{grid}*.nc" + OBS: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: /this/is/overwritten + dirname_template: "Tier{tier}/{dataset}" + filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" + OBS4MIPS: + data: + local: + type: esmvalcore.io.local.LocalDataSource + rootpath: /this/is/overwritten + dirname_template: "Tier{tier}/{dataset}" + filename_template: "{short_name}_*.nc" + +# This is overwritten by the environment variable +max_parallel_tasks: null + +# These are other sections provided by default configuration files from ESMValTool +search_data: quick +auxiliary_data_dir: "" # Changed from default as we don't use this +log_level: info +exit_on_warning: false # Only used in NCL diagnostic scripts. +output_file_type: png +remove_preproc_dir: false # Changed from default as use will likely Cylc clean +compress_netcdf: false +save_intermediary_cubes: false +profile_diagnostic: false \ No newline at end of file diff --git a/CMEW/opt/rose-suite-metoffice.conf b/CMEW/opt/rose-suite-metoffice.conf index fe3adbf5..f80ba58c 100644 --- a/CMEW/opt/rose-suite-metoffice.conf +++ b/CMEW/opt/rose-suite-metoffice.conf @@ -5,10 +5,9 @@ AUTOASSESS=true AUTOASSESS_TITLE="AUTOASSESS TITLE" CDDS_VERSION="3.3.3" -DRS_CMIP6="BADC" -DRS_OBS4MIPS="default" -ESMVALTOOL_MODULE_NAME="scitools/community/esmvaltool/2.13.0" +ESMVALTOOL_MODULE_NAME="scitools/community/esmvaltool/2.14.0" MIP_TABLE_DIR="~cdds/etc/mip_tables/GCModelDev/0.0.25" ROOTPATH_CMIP6="/data/users/managecmip/champ/CMIP6" +ROOTPATH_OBS="/data/users/esmval/ESMValTool/obs" ROOTPATH_OBS4MIPS="/data/users/esmval/ESMValTool/temporary/obs" SITE="metoffice" From 56cbd9b9533223fdc0d927140969d2ad83e749a6 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:19:38 +0100 Subject: [PATCH 2/8] Unit tests --- .../configure_recipe/bin/configure_recipe.py | 66 +++++----- .../bin/test_configure_recipe.py | 120 ++++++++---------- .../etc/esmvaltool_config.yml | 2 +- CMEW/app/unittest/kgo/esmval_defaults.yml | 43 +++++++ .../unittest/mock_data/esmval_defaults.yml | 43 +++++++ CMEW/flow.cylc | 6 +- 6 files changed, 177 insertions(+), 103 deletions(-) create mode 100644 CMEW/app/unittest/kgo/esmval_defaults.yml create mode 100644 CMEW/app/unittest/mock_data/esmval_defaults.yml diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index e3007014..de1c38fd 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -22,12 +22,13 @@ def main(): """ # Retrieve relevant environment variables values = retrieve_values_from_task_env() + defaults = retrieve_default_settings() logger.info("Retrieving values") # Create a single configuration file user_config_path = values["USER_CONFIG_PATH"] logger.info("Creating user config") - user_config_contents = create_user_config(values) + user_config_contents = create_user_config(defaults, values) # Write the file out ensure_parent_dir(user_config_path) @@ -48,9 +49,7 @@ def retrieve_values_from_task_env(): """ values_from_task_env = { "CYLC_WORKFLOW_SHARE_DIR": os.environ["CYLC_WORKFLOW_SHARE_DIR"], - "DEV_CONFIG_PATH": os.environ["DEV_CONFIG_PATH"], "MAX_PARALLEL_TASKS": os.environ["MAX_PARALLEL_TASKS"], - "MIP_TABLE_DIR": os.environ["MIP_TABLE_DIR"], "OUTPUT_DIR": os.environ["OUTPUT_DIR"], "ROOTPATH_CMIP6": os.environ["ROOTPATH_CMIP6"], "ROOTPATH_OBS": os.environ["ROOTPATH_OBS"], @@ -61,11 +60,22 @@ def retrieve_values_from_task_env(): return values_from_task_env -def retrieve_default_values(): +def retrieve_default_settings(): + """ + Return the contents of the default configuration file. + Returns + ------- + dict + The contents of the default configuration file. + """ + with open(os.environ["ESMVAL_CONFIG_DEFAULT_PATH"], "r") as f: + defaults = yaml.safe_load(f) + logger.debug("Default values:\n%s", defaults) + return defaults -def create_user_config(values=None): +def create_user_config(defaults, values=None): """ Return the contents of the user configuration file. @@ -81,31 +91,27 @@ def create_user_config(values=None): """ values = values or {} - esmval = None - if "CYLC_WORKFLOW_SHARE_DIR" in values: - esmval = os.path.join( - values["CYLC_WORKFLOW_SHARE_DIR"], - "work", - "GCModelDev", - ) - logger.debug("esmval: %s", esmval) - - # Note that 'auxiliary_data_dir' and 'download_dir' - # are set to empty values and cannot currently be - # configured. However, 'download_dir' is used only when using the - # automatic download feature via ESMValTool (which we do not intend - # to use here). - # 'auxiliary_data_dir' is used by some recipes to look for - # additional datasets, so may need to be configured in the future. - - user_config_file_contents = { - "max_parallel_tasks": max_parallel_tasks, - "output_dir": values.get("OUTPUT_DIR"), - - "CMIP6": values.get("ROOTPATH_CMIP6"), - "OBS": values.get("ROOTPATH_OBS"), - "OBS4MIPs": values.get("ROOTPATH_OBS4MIPS"), - "ESMVal": esmval, + # Get filepaths from values and construct CMEW specific filepath + fp_dict = { + "CMIP6": values.get("ROOTPATH_CMIP6"), + "OBS": values.get("ROOTPATH_OBS"), + "Obs4MIPS": values.get("ROOTPATH_OBS4MIPS"), + "ESMVal": os.path.join( + values["CYLC_WORKFLOW_SHARE_DIR"], + "work", + "GCModelDev", + ), + } + logger.debug("ESMVal: %s", fp_dict["ESMVal"]) + + # Set up the dictionary with defaults + user_config_file_contents = defaults + + # Overwrite values with those from environment + user_config_file_contents["output_dir"] = values.get("OUTPUT_DIR") + user_config_file_contents["max_parallel_tasks"] = values.get("MAX_PARALLEL_TASKS") + for project in fp_dict: + user_config_file_contents["projects"][project]["data"]["local"]["rootpath"] = fp_dict[project] logger.debug("User config file contents:\n%s", user_config_file_contents) return user_config_file_contents diff --git a/CMEW/app/configure_recipe/bin/test_configure_recipe.py b/CMEW/app/configure_recipe/bin/test_configure_recipe.py index 5421ffae..086004cb 100644 --- a/CMEW/app/configure_recipe/bin/test_configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/test_configure_recipe.py @@ -2,83 +2,67 @@ # (C) Crown Copyright 2024-2026, Met Office. # The LICENSE.md file contains full licensing details. import pytest -from configure_recipe import create_user_config, create_developer_config +from pathlib import Path +import yaml +from configure_recipe import retrieve_values_from_task_env, retrieve_default_settings, create_user_config +# For inputs and expected outputs +expected_values = { + "CYLC_WORKFLOW_SHARE_DIR": "/fake/workflow/share", + "MAX_PARALLEL_TASKS": "99", + "OUTPUT_DIR": "/fake/output/dir", + "ROOTPATH_CMIP6": "/fake/cmip6/data", + "ROOTPATH_OBS": "/fake/obs/data", + "ROOTPATH_OBS4MIPS": "/fake/obs4mips/data", + "USER_CONFIG_PATH": "/path/to/write/config/to", +} -@pytest.mark.parametrize( - "input_key, output_key, expected", - [ - (None, "remove_preproc_dir", False), - ], + # For input and outputs +path_to_mock_defaults= str( + Path(__file__).parent.parent.parent + / "unittest" + / "mock_data" + / "esmval_defaults.yml" ) -def test_create_user_config_file_single_values( - input_key, output_key, expected -): - if input_key is None: - test_values = None - else: - test_values = {input_key: expected} - config_values = create_user_config(test_values) - actual = config_values[output_key] - assert actual == expected - - -@pytest.mark.parametrize( - "input_key, output_outer_key, output_inner_key, expected", - [ - ("ROOTPATH_CMIP6", "rootpath", "CMIP6", "cmip_rootpath"), - ("ESMVal", "drs", "ESMVal", "BADC"), - ], + # For checking outputs +path_to_kgo_config = str( + Path(__file__).parent.parent.parent + / "unittest" + / "kgo" + / "esmval_defaults.yml" ) -def test_create_user_config_file_nested_values( - input_key, output_outer_key, output_inner_key, expected -): - if input_key is None: - test_values = None - else: - test_values = {input_key: expected} - config_values = create_user_config(test_values) - actual = config_values[output_outer_key][output_inner_key] - assert actual == expected +@pytest.fixture +def mock_env_vars(monkeypatch): + monkeypatch.setenv("CYLC_WORKFLOW_SHARE_DIR", "/fake/workflow/share") + monkeypatch.setenv("MAX_PARALLEL_TASKS", "99") + monkeypatch.setenv("OUTPUT_DIR", "/fake/output/dir") + monkeypatch.setenv("ROOTPATH_CMIP6", "/fake/cmip6/data") + monkeypatch.setenv("ROOTPATH_OBS", "/fake/obs/data") + monkeypatch.setenv("ROOTPATH_OBS4MIPS", "/fake/obs4mips/data") + monkeypatch.setenv("USER_CONFIG_PATH", "/path/to/write/config/to") + monkeypatch.setenv("ESMVAL_CONFIG_DEFAULT_PATH", path_to_mock_defaults) -@pytest.mark.parametrize( - "input_key, output_outer_key, output_inner_key, expected", - [ - ("MIP_TABLE_DIR", "custom", "cmor_path", "test_mip_tables"), - ("ESMVal", "ESMVal", "cmor_type", "CMIP6"), - ("CMIP6", "CMIP6", "cmor_type", "CMIP6"), - ("obs4MIPs", "obs4MIPs", "cmor_path", "obs4mips"), - ], -) -def test_create_developer_config_nested_values( - input_key, output_outer_key, output_inner_key, expected -): - if input_key == "MIP_TABLE_DIR": - test_values = {input_key: expected} - else: - test_values = {"MIP_TABLE_DIR": "test_mip_tables"} - config_values = create_developer_config(test_values) - actual = config_values[output_outer_key][output_inner_key] - assert actual == expected +def test_retrieve_values_from_task_env(mock_env_vars): + expected = expected_values + actual = retrieve_values_from_task_env() + assert expected == actual -@pytest.mark.parametrize( - "output_outer_key, output_inner_key, expected", - [ - ("ESMVal", "cmor_strict", True), - ("ESMVal", "cmor_default_table_prefix", "GCModelDev_"), - ("obs4MIPs", "cmor_default_table_prefix", "obs4MIPs_"), - ], -) -def test_create_developer_config_fixed_values( - output_outer_key, output_inner_key, expected -): - test_values = {"MIP_TABLE_DIR": "test_mip_tables"} +# I'm unconvinced that this can / should be tested in this way +# as I want it as input for the next one testing at all seems better than not +def test_retrieve_default_settings(mock_env_vars): + with open(path_to_mock_defaults, "r") as f: + expected = yaml.safe_load(f) + actual = retrieve_default_settings() + assert expected == actual + - config_values = create_developer_config(test_values) - actual = config_values[output_outer_key][output_inner_key] - assert actual == expected +def test_create_user_config(mock_env_vars): + with open(path_to_kgo_config, "r") as f: + expected = yaml.safe_load(f) + actual = create_user_config(retrieve_default_settings(), expected_values) + assert expected == actual diff --git a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml index 36760c4e..7654207d 100644 --- a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml +++ b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml @@ -28,7 +28,7 @@ projects: rootpath: /this/is/overwritten dirname_template: "Tier{tier}/{dataset}" filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" - OBS4MIPS: + Obs4MIPS: data: local: type: esmvalcore.io.local.LocalDataSource diff --git a/CMEW/app/unittest/kgo/esmval_defaults.yml b/CMEW/app/unittest/kgo/esmval_defaults.yml new file mode 100644 index 00000000..a572a615 --- /dev/null +++ b/CMEW/app/unittest/kgo/esmval_defaults.yml @@ -0,0 +1,43 @@ +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. + +--- +# Fake destination directory +output_dir: /fake/output/dir + +# Data filepath structures +projects: + ESMVal: + data: + local: + type: data_type + rootpath: /fake/workflow/share/work/GCModelDev + dirname_template: "{dirname}/{1}" + filename_template: "{filename}_{1}*.nc" + CMIP6: + data: + local: + type: data_type + rootpath: /fake/cmip6/data + dirname_template: "{dirname}/{2}" + filename_template: "{filename}_{2}*.nc" + OBS: + data: + local: + type: data_type + rootpath: /fake/obs/data + dirname_template: "{dirname}/{3}" + filename_template: "{filename}_{3}*.nc" + Obs4MIPS: + data: + local: + type: edata_type + rootpath: /fake/obs4mips/data + dirname_template: "{dirname}/{3}" + filename_template: "{filename}_{3}*.nc" + +# This is overwritten by the environment variable +max_parallel_tasks: '99' + +# Static section +other_key: some_value \ No newline at end of file diff --git a/CMEW/app/unittest/mock_data/esmval_defaults.yml b/CMEW/app/unittest/mock_data/esmval_defaults.yml new file mode 100644 index 00000000..123696bd --- /dev/null +++ b/CMEW/app/unittest/mock_data/esmval_defaults.yml @@ -0,0 +1,43 @@ +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. + +--- +# Fake destination directory +output_dir: /this/is/overwritten + +# Data filepath structures +projects: + ESMVal: + data: + local: + type: data_type + rootpath: /this/is/overwritten + dirname_template: "{dirname}/{1}" + filename_template: "{filename}_{1}*.nc" + CMIP6: + data: + local: + type: data_type + rootpath: /this/is/overwritten + dirname_template: "{dirname}/{2}" + filename_template: "{filename}_{2}*.nc" + OBS: + data: + local: + type: data_type + rootpath: /this/is/overwritten + dirname_template: "{dirname}/{3}" + filename_template: "{filename}_{3}*.nc" + Obs4MIPS: + data: + local: + type: edata_type + rootpath: /this/is/overwritten + dirname_template: "{dirname}/{3}" + filename_template: "{filename}_{3}*.nc" + +# This is overwritten by the environment variable +max_parallel_tasks: null + +# Static section +other_key: some_value \ No newline at end of file diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index 9f9438b8..5699f1b3 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -47,7 +47,6 @@ ESMVALTOOL_MODULE_NAME = {{ ESMVALTOOL_MODULE_NAME }} ESMVALTOOL_CONFIG_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/etc/esmvaltool_config USER_CONFIG_PATH = ${ESMVALTOOL_CONFIG_DIR}/config-user.yml - DEV_CONFIG_PATH = ${CYLC_WORKFLOW_SHARE_DIR}/etc/config-developer.yml OUTPUT_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/cycle/${CYLC_TASK_CYCLE_POINT} SHARE_DATA_CDDS = ${CYLC_WORKFLOW_SHARE_DIR}/data/cdds ROOT_PROC_DIR = ${SHARE_DATA_CDDS}/proc @@ -91,11 +90,10 @@ [[configure_recipe]] [[[environment]]] - DRS_CMIP6 = {{ DRS_CMIP6 }} - DRS_OBS4MIPS = {{ DRS_OBS4MIPS }} + ESMVAL_CONFIG_DEFAULT_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_recipe/etc/esmvaltool_config.yml MAX_PARALLEL_TASKS = {{ MAX_PARALLEL_TASKS }} - MIP_TABLE_DIR = {{ MIP_TABLE_DIR }} ROOTPATH_CMIP6 = {{ ROOTPATH_CMIP6 }} + ROOTPATH_OBS = {{ ROOTPATH_OBS }} ROOTPATH_OBS4MIPS = {{ ROOTPATH_OBS4MIPS }} [[configure_for]] From 9129c3a6e9851b46ad99287821f39d1690c2a733 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:22:17 +0100 Subject: [PATCH 3/8] pre-commit --- .../app/configure_recipe/bin/configure_recipe.py | 16 ++++++++++------ .../bin/test_configure_recipe.py | 12 ++++++++---- .../configure_recipe/etc/esmvaltool_config.yml | 2 +- CMEW/app/unittest/kgo/esmval_defaults.yml | 2 +- CMEW/app/unittest/mock_data/esmval_defaults.yml | 2 +- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index de1c38fd..ec39d54d 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -97,10 +97,10 @@ def create_user_config(defaults, values=None): "OBS": values.get("ROOTPATH_OBS"), "Obs4MIPS": values.get("ROOTPATH_OBS4MIPS"), "ESMVal": os.path.join( - values["CYLC_WORKFLOW_SHARE_DIR"], - "work", - "GCModelDev", - ), + values["CYLC_WORKFLOW_SHARE_DIR"], + "work", + "GCModelDev", + ), } logger.debug("ESMVal: %s", fp_dict["ESMVal"]) @@ -109,9 +109,13 @@ def create_user_config(defaults, values=None): # Overwrite values with those from environment user_config_file_contents["output_dir"] = values.get("OUTPUT_DIR") - user_config_file_contents["max_parallel_tasks"] = values.get("MAX_PARALLEL_TASKS") + user_config_file_contents["max_parallel_tasks"] = values.get( + "MAX_PARALLEL_TASKS" + ) for project in fp_dict: - user_config_file_contents["projects"][project]["data"]["local"]["rootpath"] = fp_dict[project] + user_config_file_contents["projects"][project]["data"]["local"][ + "rootpath" + ] = fp_dict[project] logger.debug("User config file contents:\n%s", user_config_file_contents) return user_config_file_contents diff --git a/CMEW/app/configure_recipe/bin/test_configure_recipe.py b/CMEW/app/configure_recipe/bin/test_configure_recipe.py index 086004cb..0a528598 100644 --- a/CMEW/app/configure_recipe/bin/test_configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/test_configure_recipe.py @@ -4,7 +4,11 @@ import pytest from pathlib import Path import yaml -from configure_recipe import retrieve_values_from_task_env, retrieve_default_settings, create_user_config +from configure_recipe import ( + retrieve_values_from_task_env, + retrieve_default_settings, + create_user_config, +) # For inputs and expected outputs expected_values = { @@ -17,15 +21,15 @@ "USER_CONFIG_PATH": "/path/to/write/config/to", } - # For input and outputs -path_to_mock_defaults= str( +# For input and outputs +path_to_mock_defaults = str( Path(__file__).parent.parent.parent / "unittest" / "mock_data" / "esmval_defaults.yml" ) - # For checking outputs +# For checking outputs path_to_kgo_config = str( Path(__file__).parent.parent.parent / "unittest" diff --git a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml index 7654207d..530b4dca 100644 --- a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml +++ b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml @@ -48,4 +48,4 @@ output_file_type: png remove_preproc_dir: false # Changed from default as use will likely Cylc clean compress_netcdf: false save_intermediary_cubes: false -profile_diagnostic: false \ No newline at end of file +profile_diagnostic: false diff --git a/CMEW/app/unittest/kgo/esmval_defaults.yml b/CMEW/app/unittest/kgo/esmval_defaults.yml index a572a615..ae0237cb 100644 --- a/CMEW/app/unittest/kgo/esmval_defaults.yml +++ b/CMEW/app/unittest/kgo/esmval_defaults.yml @@ -40,4 +40,4 @@ projects: max_parallel_tasks: '99' # Static section -other_key: some_value \ No newline at end of file +other_key: some_value diff --git a/CMEW/app/unittest/mock_data/esmval_defaults.yml b/CMEW/app/unittest/mock_data/esmval_defaults.yml index 123696bd..a549990c 100644 --- a/CMEW/app/unittest/mock_data/esmval_defaults.yml +++ b/CMEW/app/unittest/mock_data/esmval_defaults.yml @@ -40,4 +40,4 @@ projects: max_parallel_tasks: null # Static section -other_key: some_value \ No newline at end of file +other_key: some_value From 600caa6050ab4ac8a768540f0211bc83764ccea5 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:29:35 +0100 Subject: [PATCH 4/8] Correcting capitalisation --- CMEW/app/configure_recipe/bin/configure_recipe.py | 2 +- CMEW/app/configure_recipe/etc/esmvaltool_config.yml | 2 +- CMEW/app/unittest/kgo/esmval_defaults.yml | 2 +- CMEW/app/unittest/mock_data/esmval_defaults.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index ec39d54d..bf0ad90a 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -95,7 +95,7 @@ def create_user_config(defaults, values=None): fp_dict = { "CMIP6": values.get("ROOTPATH_CMIP6"), "OBS": values.get("ROOTPATH_OBS"), - "Obs4MIPS": values.get("ROOTPATH_OBS4MIPS"), + "Obs4MIPs": values.get("ROOTPATH_OBS4MIPS"), "ESMVal": os.path.join( values["CYLC_WORKFLOW_SHARE_DIR"], "work", diff --git a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml index 530b4dca..da05dc5b 100644 --- a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml +++ b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml @@ -28,7 +28,7 @@ projects: rootpath: /this/is/overwritten dirname_template: "Tier{tier}/{dataset}" filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" - Obs4MIPS: + Obs4MIPs: data: local: type: esmvalcore.io.local.LocalDataSource diff --git a/CMEW/app/unittest/kgo/esmval_defaults.yml b/CMEW/app/unittest/kgo/esmval_defaults.yml index ae0237cb..84fb212a 100644 --- a/CMEW/app/unittest/kgo/esmval_defaults.yml +++ b/CMEW/app/unittest/kgo/esmval_defaults.yml @@ -28,7 +28,7 @@ projects: rootpath: /fake/obs/data dirname_template: "{dirname}/{3}" filename_template: "{filename}_{3}*.nc" - Obs4MIPS: + Obs4MIPs: data: local: type: edata_type diff --git a/CMEW/app/unittest/mock_data/esmval_defaults.yml b/CMEW/app/unittest/mock_data/esmval_defaults.yml index a549990c..0553c6bc 100644 --- a/CMEW/app/unittest/mock_data/esmval_defaults.yml +++ b/CMEW/app/unittest/mock_data/esmval_defaults.yml @@ -28,7 +28,7 @@ projects: rootpath: /this/is/overwritten dirname_template: "{dirname}/{3}" filename_template: "{filename}_{3}*.nc" - Obs4MIPS: + Obs4MIPs: data: local: type: edata_type From a4f886d3661b07b7d6bb3bb07f2d9797e46243e2 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:42:17 +0100 Subject: [PATCH 5/8] STILL correcting capitalisation --- CMEW/app/configure_recipe/etc/esmvaltool_config.yml | 2 +- CMEW/app/unittest/kgo/esmval_defaults.yml | 2 +- CMEW/app/unittest/mock_data/esmval_defaults.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml index da05dc5b..f4764dfc 100644 --- a/CMEW/app/configure_recipe/etc/esmvaltool_config.yml +++ b/CMEW/app/configure_recipe/etc/esmvaltool_config.yml @@ -28,7 +28,7 @@ projects: rootpath: /this/is/overwritten dirname_template: "Tier{tier}/{dataset}" filename_template: "{project}_{dataset}_{type}_{version}_{mip}_{short_name}[_.]*nc" - Obs4MIPs: + obs4MIPs: data: local: type: esmvalcore.io.local.LocalDataSource diff --git a/CMEW/app/unittest/kgo/esmval_defaults.yml b/CMEW/app/unittest/kgo/esmval_defaults.yml index 84fb212a..6c476e61 100644 --- a/CMEW/app/unittest/kgo/esmval_defaults.yml +++ b/CMEW/app/unittest/kgo/esmval_defaults.yml @@ -28,7 +28,7 @@ projects: rootpath: /fake/obs/data dirname_template: "{dirname}/{3}" filename_template: "{filename}_{3}*.nc" - Obs4MIPs: + obs4MIPs: data: local: type: edata_type diff --git a/CMEW/app/unittest/mock_data/esmval_defaults.yml b/CMEW/app/unittest/mock_data/esmval_defaults.yml index 0553c6bc..15f2a5bd 100644 --- a/CMEW/app/unittest/mock_data/esmval_defaults.yml +++ b/CMEW/app/unittest/mock_data/esmval_defaults.yml @@ -28,7 +28,7 @@ projects: rootpath: /this/is/overwritten dirname_template: "{dirname}/{3}" filename_template: "{filename}_{3}*.nc" - Obs4MIPs: + obs4MIPs: data: local: type: edata_type From b18e0a79b451c14bf05df48dcbfb077462b10799 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:43:04 +0100 Subject: [PATCH 6/8] OM STILL correcting capitalisation --- CMEW/app/configure_recipe/bin/configure_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index bf0ad90a..c9ea6b1d 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -95,7 +95,7 @@ def create_user_config(defaults, values=None): fp_dict = { "CMIP6": values.get("ROOTPATH_CMIP6"), "OBS": values.get("ROOTPATH_OBS"), - "Obs4MIPs": values.get("ROOTPATH_OBS4MIPS"), + "obs4MIPs": values.get("ROOTPATH_OBS4MIPS"), "ESMVal": os.path.join( values["CYLC_WORKFLOW_SHARE_DIR"], "work", From 36e59e47f222f65d60f1d265a27641b41607d1f5 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:49:48 +0100 Subject: [PATCH 7/8] int not str --- CMEW/app/configure_recipe/bin/configure_recipe.py | 4 ++-- CMEW/app/unittest/kgo/esmval_defaults.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index c9ea6b1d..0ce40e0a 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -109,9 +109,9 @@ def create_user_config(defaults, values=None): # Overwrite values with those from environment user_config_file_contents["output_dir"] = values.get("OUTPUT_DIR") - user_config_file_contents["max_parallel_tasks"] = values.get( + user_config_file_contents["max_parallel_tasks"] = int(values.get( "MAX_PARALLEL_TASKS" - ) + )) for project in fp_dict: user_config_file_contents["projects"][project]["data"]["local"][ "rootpath" diff --git a/CMEW/app/unittest/kgo/esmval_defaults.yml b/CMEW/app/unittest/kgo/esmval_defaults.yml index 6c476e61..d6076f6c 100644 --- a/CMEW/app/unittest/kgo/esmval_defaults.yml +++ b/CMEW/app/unittest/kgo/esmval_defaults.yml @@ -37,7 +37,7 @@ projects: filename_template: "{filename}_{3}*.nc" # This is overwritten by the environment variable -max_parallel_tasks: '99' +max_parallel_tasks: 99 # Static section other_key: some_value From 987150946344d31a135687e3f816f4635abf95cf Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Fri, 12 Jun 2026 17:52:04 +0100 Subject: [PATCH 8/8] pre-commit again --- CMEW/app/configure_recipe/bin/configure_recipe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMEW/app/configure_recipe/bin/configure_recipe.py b/CMEW/app/configure_recipe/bin/configure_recipe.py index 0ce40e0a..a292dd93 100755 --- a/CMEW/app/configure_recipe/bin/configure_recipe.py +++ b/CMEW/app/configure_recipe/bin/configure_recipe.py @@ -109,9 +109,9 @@ def create_user_config(defaults, values=None): # Overwrite values with those from environment user_config_file_contents["output_dir"] = values.get("OUTPUT_DIR") - user_config_file_contents["max_parallel_tasks"] = int(values.get( - "MAX_PARALLEL_TASKS" - )) + user_config_file_contents["max_parallel_tasks"] = int( + values.get("MAX_PARALLEL_TASKS") + ) for project in fp_dict: user_config_file_contents["projects"][project]["data"]["local"][ "rootpath"