From b89d990a847bf9f2e19ffe117bef80e7a3c66097 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 09:59:06 +0100 Subject: [PATCH 01/13] configure_standardise YAMLs --- .../bin/create_request_file.py | 28 ++++++-------- .../bin/create_variables_file.py | 13 +++---- .../bin/request_defaults_config.py | 38 +++++++++++++++++++ .../bin/streams_config.py | 28 ++++++++++++++ .../etc/request_defaults.yml | 22 ----------- .../app/configure_standardise/etc/streams.yml | 17 --------- CMEW/flow.cylc | 2 - 7 files changed, 82 insertions(+), 66 deletions(-) create mode 100644 CMEW/app/configure_standardise/bin/request_defaults_config.py create mode 100644 CMEW/app/configure_standardise/bin/streams_config.py delete mode 100644 CMEW/app/configure_standardise/etc/request_defaults.yml delete mode 100644 CMEW/app/configure_standardise/etc/streams.yml diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index f4e9a3e1..37fb443e 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -25,12 +25,10 @@ def load_request_defaults(): dict CDDS request configuration default settings. """ - # Get path to default settings - defaults = os.environ["REQUEST_DEFAULTS_PATH"] + # CDDS default values are in a file in the same directory + import request_defaults_config - # Read the defaults - with open(defaults, "r") as f: - config = yaml.safe_load(f) + config = request_defaults_config.request_defaults logger.debug( "Default config:\n%s", @@ -41,27 +39,24 @@ def load_request_defaults(): def list_streams(): """ - Lists all streams in the ../etc/streams.yml file. + Lists all streams in the streams_config.py file. Returns ------- str Space separated list of all streams. """ - # Get path to stream mappings - streams_config = os.environ["STREAM_CONFIG_PATH"] + # Stream mappings are in the same directory + import streams_config - # Read the stream mappings - with open(streams_config, "r") as f: - config = yaml.safe_load(f) - logger.debug( - "Stream config:\n%s", - config, - ) + config = streams_config.streams_dict + logger.debug("Stream information:\n%s", config) # List all streams (keys) all_streams = [] for stream in config: + # For substreams we only want the first part + stream = stream.split("/")[0] all_streams.append(stream) # Return as a space separated list @@ -103,8 +98,7 @@ def create_request(model_run): request = {} request["metadata"] = { **defaults["metadata"], - # The internal dictionary replaces the T with a space - "base_date": defaults["metadata"]["base_date"].isoformat(), + "base_date": defaults["metadata"]["base_date"], "calendar": dataset_dict["calendar"], "experiment_id": dataset_dict["experiment_id"], "institution_id": dataset_dict["institute"], diff --git a/CMEW/app/configure_standardise/bin/create_variables_file.py b/CMEW/app/configure_standardise/bin/create_variables_file.py index ad541139..6eb1a0c1 100755 --- a/CMEW/app/configure_standardise/bin/create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/create_variables_file.py @@ -5,7 +5,6 @@ Generates the variables.txt file from the ESMValTool recipe. """ import os -import yaml import sys import logging @@ -46,20 +45,18 @@ def combine_variable_lists(directory): def load_stream_dict(): """ - Loads stream information from the ../etc/streams.yml file. + Loads stream information from the streams_config.py file. Returns ------- dict A mapping of pre-defined streams to their associated variables """ - # Get path to stream mappings - streams_config = os.environ["STREAM_CONFIG_PATH"] - logger.debug("Reading streams from %s", streams_config) + # Stream mappings are in the same directory + import streams_config - # Read the stream mappings - with open(streams_config, "r") as f: - config = yaml.safe_load(f) + config = streams_config.streams_dict + logger.debug("Stream information:\n%s", config) # Return the whole dictionary return config diff --git a/CMEW/app/configure_standardise/bin/request_defaults_config.py b/CMEW/app/configure_standardise/bin/request_defaults_config.py new file mode 100644 index 00000000..0095360b --- /dev/null +++ b/CMEW/app/configure_standardise/bin/request_defaults_config.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. +""" +Default information to write into the CDDS request file. +""" +request_defaults = { + "metadata": { + "base_date": "1850-01-01T00:00:00", + "branch_method": "no parent", + "license": ( + "GCModelDev model data is licensed under the " + "Open Government License v3 " + "(https://www.nationalarchives.gov.uk/" + "doc/open-government-licence/version/3/)" + ), + "mip": "ESMVal", + "mip_era": "GCModelDev", + "model_type": "AGCM AER", + }, + "common": { + "mode": "relaxed", + "package": "round-1", + }, + "data": { + "mass_data_class": "crum", + "model_workflow_branch": "trunk", + "model_workflow_revision": "not used except with data request", + }, + "misc": { + "atmos_timestep": 1200, + }, + "conversion": { + "mip_convert_plugin": "HadGEM3", + "skip_archive": True, + "cylc_args": "--no-detach -v", + }, +} diff --git a/CMEW/app/configure_standardise/bin/streams_config.py b/CMEW/app/configure_standardise/bin/streams_config.py new file mode 100644 index 00000000..2b5ccbbc --- /dev/null +++ b/CMEW/app/configure_standardise/bin/streams_config.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. +""" +Information on which streams contain variables from different MIP tables. +""" +streams_dict = { + "apm": [ + "Amon/hfls", + "Amon/hfss", + "Amon/rlds", + "Amon/rlut", + "Amon/rlutcs", + "Amon/rsds", + "Amon/rsdt", + "Amon/rsut", + "Amon/rsutcs", + "Amon/tas", + "Emon/rls", + "Emon/rss", + ], + "inm": [ + "SImon/siconc", + ], + "onm/grid-T": [ + "Omon/tos", + ], +} diff --git a/CMEW/app/configure_standardise/etc/request_defaults.yml b/CMEW/app/configure_standardise/etc/request_defaults.yml deleted file mode 100644 index be203570..00000000 --- a/CMEW/app/configure_standardise/etc/request_defaults.yml +++ /dev/null @@ -1,22 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -metadata: - base_date: 1850-01-01T00:00:00 - branch_method: no parent - license: GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) - mip: ESMVal - mip_era: GCModelDev - model_type: AGCM AER -common: - mode: relaxed - package: round-1 -data: - mass_data_class: crum - model_workflow_branch: trunk - model_workflow_revision: not used except with data request -misc: - atmos_timestep: 1200 -conversion: - mip_convert_plugin: HadGEM3 - skip_archive: True - cylc_args: --no-detach -v diff --git a/CMEW/app/configure_standardise/etc/streams.yml b/CMEW/app/configure_standardise/etc/streams.yml deleted file mode 100644 index 4fb1f357..00000000 --- a/CMEW/app/configure_standardise/etc/streams.yml +++ /dev/null @@ -1,17 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -apm: -- Amon/hfls -- Amon/hfss -- Amon/rlds -- Amon/rlut -- Amon/rlutcs -- Amon/rsds -- Amon/rsdt -- Amon/rsut -- Amon/rsutcs -- Amon/tas -- Emon/rls -- Emon/rss -inm: -- SImon/siconc diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index ba946872..fb55e471 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -84,8 +84,6 @@ CDDS_SOFTWARE_DIR = "~cdds/software" RAW_DATA_DIR = {{ RAW_DATA_DIR | default("") }} RAW_DATA_DIR_MODE = {{ RAW_DATA_DIR_MODE }} - REQUEST_DEFAULTS_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_standardise/etc/request_defaults.yml - STREAM_CONFIG_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_standardise/etc/streams.yml # Workaround for bug in CDDS: ROOT_SOFTWARE_DIR: unbound variable. ROOT_SOFTWARE_DIR = ${CDDS_SOFTWARE_DIR} CDDS_VERSION = {{ CDDS_VERSION }} From efa7ee9ca79523fc3e86a776af2db76ee73cf158 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 11:06:34 +0100 Subject: [PATCH 02/13] configure_for YAML --- CMEW/app/configure_for/bin/fetch_recipe.py | 10 ++++----- .../configure_for/bin/recipe_paths_config.py | 21 +++++++++++++++++++ .../configure_for/bin/update_recipe_file.py | 11 +++++----- CMEW/app/configure_for/etc/recipe_paths.yml | 12 ----------- CMEW/flow.cylc | 1 - 5 files changed, 30 insertions(+), 25 deletions(-) create mode 100644 CMEW/app/configure_for/bin/recipe_paths_config.py delete mode 100644 CMEW/app/configure_for/etc/recipe_paths.yml diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index c1fcd2ac..f8067071 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -3,7 +3,6 @@ # The LICENSE.md file contains full licensing details. import os import subprocess -import yaml import sys import logging @@ -30,11 +29,10 @@ def retrieve_name_and_fp(): recipe = os.environ["CYLC_TASK_PARAM_recipe"] logger.info("Fetching recipe %s", recipe) - # Load the yaml config file from ../etc - recipe_dict_fp = os.environ["RECIPE_DICT_PATH"] - logger.debug("Reading recipe dict from %s", recipe_dict_fp) - with open(recipe_dict_fp, "r") as f: - recipe_dict = yaml.safe_load(f) + # Load the yaml config file from recipe_paths_config.py + from recipe_paths_config import recipes_dict + + recipe_dict = recipes_dict logger.debug("Recipe dict:\n%s", recipe_dict) # Read specific recipe names and filepaths from the yaml config file diff --git a/CMEW/app/configure_for/bin/recipe_paths_config.py b/CMEW/app/configure_for/bin/recipe_paths_config.py new file mode 100644 index 00000000..f70be5ac --- /dev/null +++ b/CMEW/app/configure_for/bin/recipe_paths_config.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. +""" +Information about the locations of specific ESMValTool recipes. +""" +recipes_dict = { + "correlation": { + "recipe_name": "recipe_correlation.yml", + "recipe_fp": "examples/recipe_correlation.yml", + "empty_additional_datasets": True, + }, + "python": { + "recipe_name": "recipe_python.yml", + "recipe_fp": "examples/recipe_python.yml", + }, + "ref_cre": { + "recipe_name": "recipe_ref_cre.yml", + "recipe_fp": "ref/recipe_ref_cre.yml", + }, +} diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index b2108555..4e395300 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -98,7 +98,7 @@ def remove_additional_datasets(recipe): Optionally remove additional_datasets sections from an ESMValTool recipe. The option to remove additional datasets is controlled by the key - empty_additional_datasets in the YAML file at RECIPE_DICT_PATH. + empty_additional_datasets in the recipe_paths_config.py file. Parameters ---------- @@ -114,11 +114,10 @@ def remove_additional_datasets(recipe): # Look up the recipe and destination from the environment recipe_id = os.environ["CYLC_TASK_PARAM_recipe"] - # Load the yaml config file from ../etc - recipe_dict_fp = os.environ["RECIPE_DICT_PATH"] - logger.debug("Reading recipe dict from %s", recipe_dict_fp) - with open(recipe_dict_fp, "r") as f: - recipe_dict = yaml.safe_load(f) + # Load the yaml config file from recipe_paths_config.py + from recipe_paths_config import recipes_dict + + recipe_dict = recipes_dict logger.debug("Recipe dict:\n%s", recipe_dict) # Don't empty by default diff --git a/CMEW/app/configure_for/etc/recipe_paths.yml b/CMEW/app/configure_for/etc/recipe_paths.yml deleted file mode 100644 index 26b3d03e..00000000 --- a/CMEW/app/configure_for/etc/recipe_paths.yml +++ /dev/null @@ -1,12 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -correlation: - recipe_name: recipe_correlation.yml - recipe_fp: examples/recipe_correlation.yml - empty_additional_datasets: true -python: - recipe_name: recipe_python.yml - recipe_fp: examples/recipe_python.yml -ref_cre: - recipe_name: recipe_ref_cre.yml - recipe_fp: ref/recipe_ref_cre.yml diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index fb55e471..d57db030 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -73,7 +73,6 @@ [[RECIPE]] [[[environment]]] - RECIPE_DICT_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_for/etc/recipe_paths.yml RECIPE_PATH = "${CYLC_WORKFLOW_SHARE_DIR}/etc/recipe_${CYLC_TASK_PARAM_recipe}.yml" VARIABLES_LIST_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/variables_lists RECIPE_VARIABLES_PATH = ${VARIABLES_LIST_DIR}/${CYLC_TASK_PARAM_recipe}_variables.txt From 74056b90667fa98f2d5cab67c03eedc2c32e9dca Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 11:38:10 +0100 Subject: [PATCH 03/13] configure_for importlib --- CMEW/app/configure_for/bin/fetch_recipe.py | 14 +++++++------- CMEW/app/configure_for/bin/update_recipe_file.py | 14 ++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index f8067071..7ce99d9e 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -2,6 +2,7 @@ # (C) Crown Copyright 2026, Met Office. # The LICENSE.md file contains full licensing details. import os +import importlib import subprocess import sys import logging @@ -11,9 +12,9 @@ logger = logging.getLogger(filename) -def retrieve_name_and_fp(): +def retrieve_name_and_fp(recipe_paths_file): """ - Looks in recipe_paths.yml for an entry or constructs default values. + Looks in the `recipe_paths_file` for an entry or constructs default values. Uses the environment variable CYLC_TASK_PARAM_recipe as a dict key. @@ -29,10 +30,9 @@ def retrieve_name_and_fp(): recipe = os.environ["CYLC_TASK_PARAM_recipe"] logger.info("Fetching recipe %s", recipe) - # Load the yaml config file from recipe_paths_config.py - from recipe_paths_config import recipes_dict - - recipe_dict = recipes_dict + # Load the recipes config file + module = importlib.import_module(recipe_paths_file) + recipe_dict = module.recipes_dict logger.debug("Recipe dict:\n%s", recipe_dict) # Read specific recipe names and filepaths from the yaml config file @@ -53,7 +53,7 @@ def retrieve_name_and_fp(): def main(): """Fetch a recipe from ESMValTool and copy it to the recipe path.""" # Find the full name and location within ESMValTool - recipe_name, recipe_fp = retrieve_name_and_fp() + recipe_name, recipe_fp = retrieve_name_and_fp("recipe_paths_config") # Look up final destination destination_fp = os.environ["RECIPE_PATH"] diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index 4e395300..96041428 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -8,6 +8,7 @@ * User configurable variables from the Rose suite configuration """ import os +import importlib import yaml import sys import logging @@ -93,7 +94,7 @@ def add_extra_datasets(recipe, yaml_filepath): return recipe -def remove_additional_datasets(recipe): +def remove_additional_datasets(recipe, recipe_paths_file): """ Optionally remove additional_datasets sections from an ESMValTool recipe. @@ -114,10 +115,9 @@ def remove_additional_datasets(recipe): # Look up the recipe and destination from the environment recipe_id = os.environ["CYLC_TASK_PARAM_recipe"] - # Load the yaml config file from recipe_paths_config.py - from recipe_paths_config import recipes_dict - - recipe_dict = recipes_dict + # Load the recipes config file + module = importlib.import_module(recipe_paths_file) + recipe_dict = module.recipes_dict logger.debug("Recipe dict:\n%s", recipe_dict) # Don't empty by default @@ -175,7 +175,9 @@ def main(): logger.info("Amending recipe from %s", recipe_path) # Remove additional datasets if specified - amended_recipe = remove_additional_datasets(blank_recipe) + amended_recipe = remove_additional_datasets( + blank_recipe, "recipe_paths_config" + ) # Add the model runs into the datasets section of the recipe model_runs_fp = f"{os.environ['DATASETS_LIST_DIR']}/model_runs.yml" From 0e33c7ca7d904af68412fa1beea3ecf51212c6f3 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 11:54:16 +0100 Subject: [PATCH 04/13] configure_standardise importlib --- .../bin/create_request_file.py | 53 +++++++++++++------ .../bin/create_variables_file.py | 25 ++++++--- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index 37fb443e..87be9ffa 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -5,6 +5,7 @@ Generate CDDS request configuration file. """ import configparser +import importlib import os import sys from pathlib import Path @@ -16,20 +17,24 @@ logger = logging.getLogger(filename) -def load_request_defaults(): +def load_request_defaults(request_defaults_file): """ Load default values for request file. + Parameters + ---------- + request_defaults_file : str + The name of a python file in the same directory + containing the CDDS request default values. + Returns ------- dict CDDS request configuration default settings. """ - # CDDS default values are in a file in the same directory - import request_defaults_config - - config = request_defaults_config.request_defaults - + # Load the CDDS default values + module = importlib.import_module(request_defaults_file) + config = module.request_defaults logger.debug( "Default config:\n%s", config, @@ -37,19 +42,24 @@ def load_request_defaults(): return config -def list_streams(): +def list_streams(stream_info_file): """ Lists all streams in the streams_config.py file. + Parameters + ---------- + stream_info_file : str + The name of a python file in the same directory + containing information about data streams. + Returns ------- str Space separated list of all streams. """ - # Stream mappings are in the same directory - import streams_config - - config = streams_config.streams_dict + # Load the stream information dictionary + module = importlib.import_module(stream_info_file) + config = module.streams_dict logger.debug("Stream information:\n%s", config) # List all streams (keys) @@ -69,18 +79,29 @@ def list_streams(): return stream_str -def create_request(model_run): +def create_request(model_run, request_defaults_file, stream_info_file): """ Build a CDDS request configuration for a run identified by a suite_id. Uses information from the model_runs.yml file. + Parameters + ---------- + model_run : str + The suite ID as a model run identifier. + request_defaults_file : str + The name of a python file in the same directory + containing the CDDS request default values. + stream_info_file : str + The name of a python file in the same directory + containing information about data streams. + Returns ------- dict CDDS request configuration. """ - defaults = load_request_defaults() + defaults = load_request_defaults(request_defaults_file) mip_table_dir = os.environ["MIP_TABLE_DIR"] @@ -118,7 +139,7 @@ def create_request(model_run): "end_date": f"{int(dataset_dict['end_year'])+1}-01-01T00:00:00", "model_workflow_id": dataset_dict["suite_id"], # List all possible streams as CDDS just ignores ones without variables - "streams": list_streams(), + "streams": list_streams(stream_info_file), "variable_list_file": os.environ["VARIABLES_PATH"], } request["misc"] = dict(defaults["misc"]) @@ -162,7 +183,9 @@ def main(): dataset = os.environ["CYLC_TASK_PARAM_dataset"].strip() logger.info("Creating CDDS request for dataset %s", dataset) - request = create_request(dataset) + request = create_request( + dataset, "request_defaults_config", "streams_config" + ) target_path = Path(os.environ["REQUEST_PATH"]) write_request(request, target_path) diff --git a/CMEW/app/configure_standardise/bin/create_variables_file.py b/CMEW/app/configure_standardise/bin/create_variables_file.py index 6eb1a0c1..bf792319 100755 --- a/CMEW/app/configure_standardise/bin/create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/create_variables_file.py @@ -5,6 +5,7 @@ Generates the variables.txt file from the ESMValTool recipe. """ import os +import importlib import sys import logging @@ -43,39 +44,47 @@ def combine_variable_lists(directory): return variables -def load_stream_dict(): +def load_stream_dict(stream_info_file): """ Loads stream information from the streams_config.py file. + Parameters + ---------- + stream_info_file : str + The name of a python file in the same directory + containing information about data streams. + Returns ------- dict A mapping of pre-defined streams to their associated variables """ - # Stream mappings are in the same directory - import streams_config - - config = streams_config.streams_dict + # Load the stream information dictionary + module = importlib.import_module(stream_info_file) + config = module.streams_dict logger.debug("Stream information:\n%s", config) # Return the whole dictionary return config -def add_stream_to_variables(variables): +def add_stream_to_variables(variables, stream_info_file): """Add stream information to a list of variables. Parameters ---------- variables : list[str] List of variables in the format "MIP_table/variable_name" + stream_info_file : str + The name of a python file in the same directory + containing information about data streams. Returns ------- list[str] List of variables in the format "MIP_table/variable_name:stream" """ - stream_dict = load_stream_dict() + stream_dict = load_stream_dict(stream_info_file) # Using a second dictionary to avoid looping var_to_stream = { @@ -113,7 +122,7 @@ def write_variables(variables, target_path): def main(): variables = combine_variable_lists(os.environ["VARIABLES_LIST_DIR"]) - streamed_variables = add_stream_to_variables(variables) + streamed_variables = add_stream_to_variables(variables, "streams_config") variables_path = os.environ["VARIABLES_PATH"] logger.info("Writing variables file to %s", variables_path) write_variables(streamed_variables, variables_path) From dc9cc3ff521a2ae80defcd76e9f62a9a30b1f7d6 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 11:56:26 +0100 Subject: [PATCH 05/13] docstring parameters --- CMEW/app/configure_for/bin/fetch_recipe.py | 6 ++++++ CMEW/app/configure_for/bin/update_recipe_file.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index 7ce99d9e..f7c3f89e 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -18,6 +18,12 @@ def retrieve_name_and_fp(recipe_paths_file): Uses the environment variable CYLC_TASK_PARAM_recipe as a dict key. + Parameters + ---------- + recipe_paths_file : str + The name of a python file in the same directory + containing information ESMValTool recipe locations. + Returns ------- recipe_name: str diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index 96041428..6504ec4e 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -105,6 +105,9 @@ def remove_additional_datasets(recipe, recipe_paths_file): ---------- recipe: dict The content of the recipe which may have additional datasets. + recipe_paths_file : str + The name of a python file in the same directory + containing information ESMValTool recipe locations. Returns ------- From 4c875c2f75bf7ca57426ef0b93597149d09e7d61 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 15:34:23 +0100 Subject: [PATCH 06/13] actual info as default rather than file --- .../bin/{recipe_paths_config.py => config.py} | 5 +- CMEW/app/configure_for/bin/fetch_recipe.py | 16 ++--- .../configure_for/bin/update_recipe_file.py | 24 ++++---- .../{request_defaults_config.py => config.py} | 31 ++++++++-- .../bin/create_request_file.py | 58 ++++--------------- .../bin/create_variables_file.py | 37 ++---------- .../bin/streams_config.py | 28 --------- 7 files changed, 64 insertions(+), 135 deletions(-) rename CMEW/app/configure_for/bin/{recipe_paths_config.py => config.py} (88%) rename CMEW/app/configure_standardise/bin/{request_defaults_config.py => config.py} (64%) delete mode 100644 CMEW/app/configure_standardise/bin/streams_config.py diff --git a/CMEW/app/configure_for/bin/recipe_paths_config.py b/CMEW/app/configure_for/bin/config.py similarity index 88% rename from CMEW/app/configure_for/bin/recipe_paths_config.py rename to CMEW/app/configure_for/bin/config.py index f70be5ac..d7c6923e 100644 --- a/CMEW/app/configure_for/bin/recipe_paths_config.py +++ b/CMEW/app/configure_for/bin/config.py @@ -1,9 +1,8 @@ #!/usr/bin/env python # (C) Crown Copyright 2026, Met Office. # The LICENSE.md file contains full licensing details. -""" -Information about the locations of specific ESMValTool recipes. -""" + +# Information about the locations of specific ESMValTool recipes. recipes_dict = { "correlation": { "recipe_name": "recipe_correlation.yml", diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index f7c3f89e..71550d17 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -2,17 +2,18 @@ # (C) Crown Copyright 2026, Met Office. # The LICENSE.md file contains full licensing details. import os -import importlib import subprocess import sys import logging +from config import recipes_dict + logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) logger = logging.getLogger(filename) -def retrieve_name_and_fp(recipe_paths_file): +def retrieve_name_and_fp(recipe_dict=recipes_dict): """ Looks in the `recipe_paths_file` for an entry or constructs default values. @@ -20,9 +21,10 @@ def retrieve_name_and_fp(recipe_paths_file): Parameters ---------- - recipe_paths_file : str - The name of a python file in the same directory - containing information ESMValTool recipe locations. + recipe_dict : dict + A dictionary with keys for recipe identifiers + for recipes which do not follow the default pattern + of names and locations within ESMValTool. Returns ------- @@ -37,8 +39,6 @@ def retrieve_name_and_fp(recipe_paths_file): logger.info("Fetching recipe %s", recipe) # Load the recipes config file - module = importlib.import_module(recipe_paths_file) - recipe_dict = module.recipes_dict logger.debug("Recipe dict:\n%s", recipe_dict) # Read specific recipe names and filepaths from the yaml config file @@ -59,7 +59,7 @@ def retrieve_name_and_fp(recipe_paths_file): def main(): """Fetch a recipe from ESMValTool and copy it to the recipe path.""" # Find the full name and location within ESMValTool - recipe_name, recipe_fp = retrieve_name_and_fp("recipe_paths_config") + recipe_name, recipe_fp = retrieve_name_and_fp() # Look up final destination destination_fp = os.environ["RECIPE_PATH"] diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index 6504ec4e..9a860fa8 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -8,10 +8,10 @@ * User configurable variables from the Rose suite configuration """ import os -import importlib import yaml import sys import logging +from config import recipes_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) @@ -94,7 +94,7 @@ def add_extra_datasets(recipe, yaml_filepath): return recipe -def remove_additional_datasets(recipe, recipe_paths_file): +def remove_additional_datasets(recipe, recipe_dict=recipes_dict): """ Optionally remove additional_datasets sections from an ESMValTool recipe. @@ -105,9 +105,10 @@ def remove_additional_datasets(recipe, recipe_paths_file): ---------- recipe: dict The content of the recipe which may have additional datasets. - recipe_paths_file : str - The name of a python file in the same directory - containing information ESMValTool recipe locations. + recipe_dict : dict + A dictionary with keys for a recipe identifier and the value + True assigned to an inner key of empty_additional_datasets + if additional datasets are to be emptied from a recipe. Returns ------- @@ -118,15 +119,12 @@ def remove_additional_datasets(recipe, recipe_paths_file): # Look up the recipe and destination from the environment recipe_id = os.environ["CYLC_TASK_PARAM_recipe"] - # Load the recipes config file - module = importlib.import_module(recipe_paths_file) - recipe_dict = module.recipes_dict - logger.debug("Recipe dict:\n%s", recipe_dict) - # Don't empty by default empty_additionals = False - # Read specific recipe names and filepaths from the yaml config file + # Read specific recipe names and filepaths from the config file + logger.debug("Recipe dict:\n%s", recipe_dict) + if recipe_id in recipe_dict: logger.debug("Using info from recipe dictionary for %s", recipe_id) if "empty_additional_datasets" in recipe_dict[recipe_id]: @@ -178,9 +176,7 @@ def main(): logger.info("Amending recipe from %s", recipe_path) # Remove additional datasets if specified - amended_recipe = remove_additional_datasets( - blank_recipe, "recipe_paths_config" - ) + amended_recipe = remove_additional_datasets(blank_recipe) # Add the model runs into the datasets section of the recipe model_runs_fp = f"{os.environ['DATASETS_LIST_DIR']}/model_runs.yml" diff --git a/CMEW/app/configure_standardise/bin/request_defaults_config.py b/CMEW/app/configure_standardise/bin/config.py similarity index 64% rename from CMEW/app/configure_standardise/bin/request_defaults_config.py rename to CMEW/app/configure_standardise/bin/config.py index 0095360b..9bcd0455 100644 --- a/CMEW/app/configure_standardise/bin/request_defaults_config.py +++ b/CMEW/app/configure_standardise/bin/config.py @@ -1,10 +1,33 @@ #!/usr/bin/env python # (C) Crown Copyright 2026, Met Office. # The LICENSE.md file contains full licensing details. -""" -Default information to write into the CDDS request file. -""" -request_defaults = { + +# Information on which streams contain variables from different MIP tables. +streams_dict = { + "apm": [ + "Amon/hfls", + "Amon/hfss", + "Amon/rlds", + "Amon/rlut", + "Amon/rlutcs", + "Amon/rsds", + "Amon/rsdt", + "Amon/rsut", + "Amon/rsutcs", + "Amon/tas", + "Emon/rls", + "Emon/rss", + ], + "inm": [ + "SImon/siconc", + ], + "onm/grid-T": [ + "Omon/tos", + ], +} + +# Default information to write into the CDDS request file. +requests_defaults = { "metadata": { "base_date": "1850-01-01T00:00:00", "branch_method": "no parent", diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index 87be9ffa..fd970b4e 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -5,52 +5,26 @@ Generate CDDS request configuration file. """ import configparser -import importlib import os import sys from pathlib import Path import yaml import logging +from config import requests_defaults, streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) logger = logging.getLogger(filename) -def load_request_defaults(request_defaults_file): - """ - Load default values for request file. - - Parameters - ---------- - request_defaults_file : str - The name of a python file in the same directory - containing the CDDS request default values. - - Returns - ------- - dict - CDDS request configuration default settings. - """ - # Load the CDDS default values - module = importlib.import_module(request_defaults_file) - config = module.request_defaults - logger.debug( - "Default config:\n%s", - config, - ) - return config - - -def list_streams(stream_info_file): +def list_streams(stream_dict=streams_dict): """ Lists all streams in the streams_config.py file. Parameters ---------- - stream_info_file : str - The name of a python file in the same directory - containing information about data streams. + stream_dict : dict + A dictionary containing information about data streams. Returns ------- @@ -58,13 +32,11 @@ def list_streams(stream_info_file): Space separated list of all streams. """ # Load the stream information dictionary - module = importlib.import_module(stream_info_file) - config = module.streams_dict - logger.debug("Stream information:\n%s", config) + logger.debug("Stream information:\n%s", stream_dict) # List all streams (keys) all_streams = [] - for stream in config: + for stream in stream_dict: # For substreams we only want the first part stream = stream.split("/")[0] all_streams.append(stream) @@ -79,7 +51,7 @@ def list_streams(stream_info_file): return stream_str -def create_request(model_run, request_defaults_file, stream_info_file): +def create_request(model_run, request_defaults=requests_defaults): """ Build a CDDS request configuration for a run identified by a suite_id. @@ -89,19 +61,15 @@ def create_request(model_run, request_defaults_file, stream_info_file): ---------- model_run : str The suite ID as a model run identifier. - request_defaults_file : str - The name of a python file in the same directory - containing the CDDS request default values. - stream_info_file : str - The name of a python file in the same directory - containing information about data streams. + request_defaults : dict + A dictionary containing the CDDS request default values. Returns ------- dict CDDS request configuration. """ - defaults = load_request_defaults(request_defaults_file) + defaults = request_defaults mip_table_dir = os.environ["MIP_TABLE_DIR"] @@ -139,7 +107,7 @@ def create_request(model_run, request_defaults_file, stream_info_file): "end_date": f"{int(dataset_dict['end_year'])+1}-01-01T00:00:00", "model_workflow_id": dataset_dict["suite_id"], # List all possible streams as CDDS just ignores ones without variables - "streams": list_streams(stream_info_file), + "streams": list_streams(), "variable_list_file": os.environ["VARIABLES_PATH"], } request["misc"] = dict(defaults["misc"]) @@ -183,9 +151,7 @@ def main(): dataset = os.environ["CYLC_TASK_PARAM_dataset"].strip() logger.info("Creating CDDS request for dataset %s", dataset) - request = create_request( - dataset, "request_defaults_config", "streams_config" - ) + request = create_request(dataset) target_path = Path(os.environ["REQUEST_PATH"]) write_request(request, target_path) diff --git a/CMEW/app/configure_standardise/bin/create_variables_file.py b/CMEW/app/configure_standardise/bin/create_variables_file.py index bf792319..e3356c1f 100755 --- a/CMEW/app/configure_standardise/bin/create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/create_variables_file.py @@ -5,9 +5,9 @@ Generates the variables.txt file from the ESMValTool recipe. """ import os -import importlib import sys import logging +from config import streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) @@ -44,48 +44,21 @@ def combine_variable_lists(directory): return variables -def load_stream_dict(stream_info_file): - """ - Loads stream information from the streams_config.py file. - - Parameters - ---------- - stream_info_file : str - The name of a python file in the same directory - containing information about data streams. - - Returns - ------- - dict - A mapping of pre-defined streams to their associated variables - """ - # Load the stream information dictionary - module = importlib.import_module(stream_info_file) - config = module.streams_dict - logger.debug("Stream information:\n%s", config) - - # Return the whole dictionary - return config - - -def add_stream_to_variables(variables, stream_info_file): +def add_stream_to_variables(variables, stream_dict=streams_dict): """Add stream information to a list of variables. Parameters ---------- variables : list[str] List of variables in the format "MIP_table/variable_name" - stream_info_file : str - The name of a python file in the same directory - containing information about data streams. + stream_dict : dict + A dictionary containing information about data streams. Returns ------- list[str] List of variables in the format "MIP_table/variable_name:stream" """ - stream_dict = load_stream_dict(stream_info_file) - # Using a second dictionary to avoid looping var_to_stream = { var: stream @@ -122,7 +95,7 @@ def write_variables(variables, target_path): def main(): variables = combine_variable_lists(os.environ["VARIABLES_LIST_DIR"]) - streamed_variables = add_stream_to_variables(variables, "streams_config") + streamed_variables = add_stream_to_variables(variables) variables_path = os.environ["VARIABLES_PATH"] logger.info("Writing variables file to %s", variables_path) write_variables(streamed_variables, variables_path) diff --git a/CMEW/app/configure_standardise/bin/streams_config.py b/CMEW/app/configure_standardise/bin/streams_config.py deleted file mode 100644 index 2b5ccbbc..00000000 --- a/CMEW/app/configure_standardise/bin/streams_config.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -""" -Information on which streams contain variables from different MIP tables. -""" -streams_dict = { - "apm": [ - "Amon/hfls", - "Amon/hfss", - "Amon/rlds", - "Amon/rlut", - "Amon/rlutcs", - "Amon/rsds", - "Amon/rsdt", - "Amon/rsut", - "Amon/rsutcs", - "Amon/tas", - "Emon/rls", - "Emon/rss", - ], - "inm": [ - "SImon/siconc", - ], - "onm/grid-T": [ - "Omon/tos", - ], -} From 13ce3bbf39cbf84b3df5e8de0ee60da16e9dc845 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 16:15:25 +0100 Subject: [PATCH 07/13] configure_for tests --- CMEW/app/configure_for/bin/fetch_recipe.py | 4 +-- .../configure_for/bin/test_fetch_recipe.py | 29 +++++++------------ .../bin/test_update_recipe_file.py | 18 +++++------- CMEW/app/unittest/mock_data/recipe_paths.yml | 6 ---- 4 files changed, 20 insertions(+), 37 deletions(-) delete mode 100644 CMEW/app/unittest/mock_data/recipe_paths.yml diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index 71550d17..92616bbf 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -38,10 +38,8 @@ def retrieve_name_and_fp(recipe_dict=recipes_dict): recipe = os.environ["CYLC_TASK_PARAM_recipe"] logger.info("Fetching recipe %s", recipe) - # Load the recipes config file + # Read specific recipe names and filepaths from the config file logger.debug("Recipe dict:\n%s", recipe_dict) - - # Read specific recipe names and filepaths from the yaml config file if recipe in recipe_dict: logger.debug("Using info from recipe dictionary for %s", recipe) recipe_name = recipe_dict[recipe]["recipe_name"] diff --git a/CMEW/app/configure_for/bin/test_fetch_recipe.py b/CMEW/app/configure_for/bin/test_fetch_recipe.py index 7829d111..74acd787 100755 --- a/CMEW/app/configure_for/bin/test_fetch_recipe.py +++ b/CMEW/app/configure_for/bin/test_fetch_recipe.py @@ -9,35 +9,28 @@ input for test_retrieve_specified, test_retrieve_defaults """ from fetch_recipe import retrieve_name_and_fp -from pathlib import Path -import pytest -@pytest.fixture -def mock_env_vars(monkeypatch): - # For adding extra datasets - monkeypatch.setenv( - "RECIPE_DICT_PATH", - str( - Path(__file__).parent.parent.parent - / "unittest" - / "mock_data" - / "recipe_paths.yml" - ), - ) +mock_recipe_dict = { + "mock_entry": { + "recipe_name": "recipe_specified_name.yml", + "recipe_fp": "subdir_1/recipe_second_name.yml", + "empty_additional_datasets": True, + }, +} -def test_retrieve_specified(mock_env_vars, monkeypatch): +def test_retrieve_specified(monkeypatch): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry") expected = "recipe_specified_name.yml", "subdir_1/recipe_second_name.yml" - actual = retrieve_name_and_fp() + actual = retrieve_name_and_fp(mock_recipe_dict) assert actual == expected -def test_retrieve_defaults(mock_env_vars, monkeypatch): +def test_retrieve_defaults(monkeypatch): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "not_here") expected = "recipe_not_here.yml", "recipe_not_here.yml" - actual = retrieve_name_and_fp() + actual = retrieve_name_and_fp(mock_recipe_dict) assert actual == expected diff --git a/CMEW/app/configure_for/bin/test_update_recipe_file.py b/CMEW/app/configure_for/bin/test_update_recipe_file.py index f27d648b..37e90d12 100644 --- a/CMEW/app/configure_for/bin/test_update_recipe_file.py +++ b/CMEW/app/configure_for/bin/test_update_recipe_file.py @@ -148,15 +148,13 @@ def test_remove_additional_datasets( path_to_recipe_additionals_removed, ): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry") - monkeypatch.setenv( - "RECIPE_DICT_PATH", - str( - Path(__file__).parent.parent.parent - / "unittest" - / "mock_data" - / "recipe_paths.yml" - ), - ) + mock_recipe_dict = { + "mock_entry": { + "recipe_name": "recipe_specified_name.yml", + "recipe_fp": "subdir_1/recipe_second_name.yml", + "empty_additional_datasets": True, + }, + } with open(path_to_recipe_additionals_removed, "r") as file_handle_1: expected = yaml.safe_load(file_handle_1) @@ -165,7 +163,7 @@ def test_remove_additional_datasets( pre_recipe = yaml.safe_load(file_handle_2) # Using str(filepath) here as update_recipe_file.py uses os, not pathlib - actual = remove_additional_datasets(pre_recipe) + actual = remove_additional_datasets(pre_recipe, mock_recipe_dict) assert actual == expected diff --git a/CMEW/app/unittest/mock_data/recipe_paths.yml b/CMEW/app/unittest/mock_data/recipe_paths.yml deleted file mode 100644 index ff5b6b04..00000000 --- a/CMEW/app/unittest/mock_data/recipe_paths.yml +++ /dev/null @@ -1,6 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -mock_entry: - recipe_name: recipe_specified_name.yml - recipe_fp: subdir_1/recipe_second_name.yml - empty_additional_datasets: true From 9329e928ae7d097ab8f41bdde325280bc1a8f0e0 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 16:46:03 +0100 Subject: [PATCH 08/13] changing unit tests --- .../bin/test_create_request_file.py | 62 ++++++++++++++++--- .../bin/test_create_variables_file.py | 26 ++++++-- CMEW/app/unittest/kgo/request_u-cw673.cfg | 2 +- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/CMEW/app/configure_standardise/bin/test_create_request_file.py b/CMEW/app/configure_standardise/bin/test_create_request_file.py index 444e1035..4b293316 100644 --- a/CMEW/app/configure_standardise/bin/test_create_request_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_request_file.py @@ -9,7 +9,25 @@ """ from pathlib import Path import configparser -from create_request_file import create_request +from create_request_file import create_request, list_streams + + +def test_list_streams(): + mock_stream_dict = { + "abc": [ + "Amon/abcd", + "Aday/efgh", + ], + "def": [ + "SImon/ijkl", + ], + "xyz/grid-A": [ + "Omon/mnop", + ], + } + expected = "abc def xyz" + actual = list_streams(mock_stream_dict) + assert actual == expected def test_create_request(monkeypatch): @@ -17,11 +35,6 @@ def test_create_request(monkeypatch): "DATASETS_LIST_DIR", str(Path(__file__).parent.parent.parent / "unittest" / "mock_data"), ) - - request_defaults_path = ( - Path(__file__).parent.parent / "etc" / "request_defaults.yml" - ) - stream_config_path = Path(__file__).parent.parent / "etc" / "streams.yml" root_proc_dir = "/path/to/proc/dir/" root_data_dir = "/path/to/data/dir/" variables_path = "/path/to/variables.txt" @@ -29,15 +42,46 @@ def test_create_request(monkeypatch): stream_id = "apm inm" monkeypatch.setenv("RAW_DATA_DIR_MODE", "use_saved") - monkeypatch.setenv("REQUEST_DEFAULTS_PATH", str(request_defaults_path)) - monkeypatch.setenv("STREAM_CONFIG_PATH", str(stream_config_path)) monkeypatch.setenv("ROOT_PROC_DIR", root_proc_dir) monkeypatch.setenv("ROOT_DATA_DIR", root_data_dir) monkeypatch.setenv("VARIABLES_PATH", variables_path) monkeypatch.setenv("MIP_TABLE_DIR", mip_table_dir) monkeypatch.setenv("STREAM_ID", stream_id) - actual_request = create_request("u-cw673") + mock_request_defaults = { + "metadata": { + "base_date": "1850-01-01T00:00:00", + "branch_method": "no parent", + "license": ( + "GCModelDev model data is licensed under the " + "Open Government License v3 " + "(https://www.nationalarchives.gov.uk/" + "doc/open-government-licence/version/3/)" + ), + "mip": "ESMVal", + "mip_era": "GCModelDev", + "model_type": "AGCM AER", + }, + "common": { + "mode": "relaxed", + "package": "round-1", + }, + "data": { + "mass_data_class": "crum", + "model_workflow_branch": "trunk", + "model_workflow_revision": "not used except with data request", + }, + "misc": { + "atmos_timestep": 1200, + }, + "conversion": { + "mip_convert_plugin": "HadGEM3", + "skip_archive": True, + "cylc_args": "--no-detach -v", + }, + } + + actual_request = create_request("u-cw673", mock_request_defaults) cfg = configparser.ConfigParser() cfg.read_dict(actual_request) actual = {section: dict(cfg[section]) for section in cfg.sections()} diff --git a/CMEW/app/configure_standardise/bin/test_create_variables_file.py b/CMEW/app/configure_standardise/bin/test_create_variables_file.py index 1683ecd1..3666638b 100644 --- a/CMEW/app/configure_standardise/bin/test_create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_variables_file.py @@ -57,9 +57,7 @@ def test_combine_variable_lists(): assert actual == expected -def test_add_stream_to_variables(monkeypatch, path_to_combined_variables): - stream_config_path = Path(__file__).parent.parent / "etc" / "streams.yml" - monkeypatch.setenv("STREAM_CONFIG_PATH", str(stream_config_path)) +def test_add_stream_to_variables(path_to_combined_variables): input = [ "Amon/hfls", "Amon/hfss", @@ -75,7 +73,27 @@ def test_add_stream_to_variables(monkeypatch, path_to_combined_variables): "Amon/tas", "SImon/siconc", ] - actual = add_stream_to_variables(input) + mock_stream_dict = { + "apm": [ + "Amon/hfls", + "Amon/hfss", + "Amon/rlds", + "Amon/rlut", + "Amon/rlutcs", + "Amon/rsds", + "Amon/rsdt", + "Amon/rsut", + "Amon/rsutcs", + "Amon/tas", + "Emon/rls", + "Emon/rss", + ], + "inm": [ + "SImon/siconc", + ], + } + + actual = add_stream_to_variables(input, mock_stream_dict) with open(path_to_combined_variables, "r") as file: expected = file.read().splitlines() diff --git a/CMEW/app/unittest/kgo/request_u-cw673.cfg b/CMEW/app/unittest/kgo/request_u-cw673.cfg index fa4ff4ee..ebf72891 100644 --- a/CMEW/app/unittest/kgo/request_u-cw673.cfg +++ b/CMEW/app/unittest/kgo/request_u-cw673.cfg @@ -26,7 +26,7 @@ model_workflow_revision = not used except with data request start_date = 1993-01-01T00:00:00 end_date = 2003-01-01T00:00:00 model_workflow_id = u-cw673 -streams = apm inm +streams = apm inm onm variable_list_file = /path/to/variables.txt [misc] From d8bccd305328f4ba0c4b8f08a7cbd997003306e2 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 17:42:52 +0100 Subject: [PATCH 09/13] Changing names for importing --- .../configure_for/bin/{config.py => config_configure_for.py} | 0 CMEW/app/configure_for/bin/fetch_recipe.py | 2 +- CMEW/app/configure_for/bin/update_recipe_file.py | 2 +- .../bin/{config.py => config_configure_standardise.py} | 0 CMEW/app/configure_standardise/bin/create_request_file.py | 2 +- CMEW/app/configure_standardise/bin/create_variables_file.py | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename CMEW/app/configure_for/bin/{config.py => config_configure_for.py} (100%) rename CMEW/app/configure_standardise/bin/{config.py => config_configure_standardise.py} (100%) diff --git a/CMEW/app/configure_for/bin/config.py b/CMEW/app/configure_for/bin/config_configure_for.py similarity index 100% rename from CMEW/app/configure_for/bin/config.py rename to CMEW/app/configure_for/bin/config_configure_for.py diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index 92616bbf..1db1afec 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -5,7 +5,7 @@ import subprocess import sys import logging -from config import recipes_dict +from config_configure_for import recipes_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index 9a860fa8..fd49be46 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -11,7 +11,7 @@ import yaml import sys import logging -from config import recipes_dict +from config_configure_for import recipes_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) diff --git a/CMEW/app/configure_standardise/bin/config.py b/CMEW/app/configure_standardise/bin/config_configure_standardise.py similarity index 100% rename from CMEW/app/configure_standardise/bin/config.py rename to CMEW/app/configure_standardise/bin/config_configure_standardise.py diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index fd970b4e..0d961d2b 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -10,7 +10,7 @@ from pathlib import Path import yaml import logging -from config import requests_defaults, streams_dict +from config_configure_standardise import requests_defaults, streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) diff --git a/CMEW/app/configure_standardise/bin/create_variables_file.py b/CMEW/app/configure_standardise/bin/create_variables_file.py index e3356c1f..3e5e9958 100755 --- a/CMEW/app/configure_standardise/bin/create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/create_variables_file.py @@ -7,7 +7,7 @@ import os import sys import logging -from config import streams_dict +from config_configure_standardise import streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) From 24f81f1b12732907b8b55f77073fcd886de9772b Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 17:46:17 +0100 Subject: [PATCH 10/13] Docstring correction --- CMEW/app/configure_for/bin/fetch_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index 1db1afec..9bcebfe4 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -15,7 +15,7 @@ def retrieve_name_and_fp(recipe_dict=recipes_dict): """ - Looks in the `recipe_paths_file` for an entry or constructs default values. + Looks in the `recipe_dict` for an entry or constructs default values. Uses the environment variable CYLC_TASK_PARAM_recipe as a dict key. From 330ddcf0c62932bec3b0d03d164609f58d2d2f7c Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 17:47:11 +0100 Subject: [PATCH 11/13] comment correction --- CMEW/app/configure_for/bin/fetch_recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index 9bcebfe4..0f866d2a 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -38,7 +38,7 @@ def retrieve_name_and_fp(recipe_dict=recipes_dict): recipe = os.environ["CYLC_TASK_PARAM_recipe"] logger.info("Fetching recipe %s", recipe) - # Read specific recipe names and filepaths from the config file + # Read specific recipe names and filepaths from the config dict logger.debug("Recipe dict:\n%s", recipe_dict) if recipe in recipe_dict: logger.debug("Using info from recipe dictionary for %s", recipe) From 39bdb52241c269c597d9a4143a8a7dce668fa734 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 17:48:39 +0100 Subject: [PATCH 12/13] Correct docstring --- CMEW/app/configure_for/bin/update_recipe_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index fd49be46..12f92d73 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -99,7 +99,7 @@ def remove_additional_datasets(recipe, recipe_dict=recipes_dict): Optionally remove additional_datasets sections from an ESMValTool recipe. The option to remove additional datasets is controlled by the key - empty_additional_datasets in the recipe_paths_config.py file. + empty_additional_datasets in the recipe_dict. Parameters ---------- From af817573af3a830d56f468196e998c8ce91f7853 Mon Sep 17 00:00:00 2001 From: Naomi Parsons Date: Mon, 20 Jul 2026 17:50:24 +0100 Subject: [PATCH 13/13] Docstring --- CMEW/app/configure_standardise/bin/create_request_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index 0d961d2b..cf7e8480 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -19,7 +19,7 @@ def list_streams(stream_dict=streams_dict): """ - Lists all streams in the streams_config.py file. + Lists only the streams in the stream_dict. Parameters ----------