diff --git a/CMEW/app/configure_for/bin/config_configure_for.py b/CMEW/app/configure_for/bin/config_configure_for.py new file mode 100644 index 00000000..d7c6923e --- /dev/null +++ b/CMEW/app/configure_for/bin/config_configure_for.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. + +# Information about the locations of specific ESMValTool recipes. +recipes_dict = { + "correlation": { + "recipe_name": "recipe_correlation.yml", + "recipe_fp": "examples/recipe_correlation.yml", + "empty_additional_datasets": True, + }, + "python": { + "recipe_name": "recipe_python.yml", + "recipe_fp": "examples/recipe_python.yml", + }, + "ref_cre": { + "recipe_name": "recipe_ref_cre.yml", + "recipe_fp": "ref/recipe_ref_cre.yml", + }, +} diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py index c1fcd2ac..0f866d2a 100755 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ b/CMEW/app/configure_for/bin/fetch_recipe.py @@ -3,21 +3,29 @@ # The LICENSE.md file contains full licensing details. import os import subprocess -import yaml import sys import logging +from config_configure_for import recipes_dict + logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) logger = logging.getLogger(filename) -def retrieve_name_and_fp(): +def retrieve_name_and_fp(recipe_dict=recipes_dict): """ - Looks in recipe_paths.yml for an entry or constructs default values. + Looks in the `recipe_dict` for an entry or constructs default values. Uses the environment variable CYLC_TASK_PARAM_recipe as a dict key. + Parameters + ---------- + recipe_dict : dict + A dictionary with keys for recipe identifiers + for recipes which do not follow the default pattern + of names and locations within ESMValTool. + Returns ------- recipe_name: str @@ -30,14 +38,8 @@ def retrieve_name_and_fp(): recipe = os.environ["CYLC_TASK_PARAM_recipe"] logger.info("Fetching recipe %s", recipe) - # Load the yaml config file from ../etc - recipe_dict_fp = os.environ["RECIPE_DICT_PATH"] - logger.debug("Reading recipe dict from %s", recipe_dict_fp) - with open(recipe_dict_fp, "r") as f: - recipe_dict = yaml.safe_load(f) + # Read specific recipe names and filepaths from the config dict logger.debug("Recipe dict:\n%s", recipe_dict) - - # Read specific recipe names and filepaths from the yaml config file if recipe in recipe_dict: logger.debug("Using info from recipe dictionary for %s", recipe) recipe_name = recipe_dict[recipe]["recipe_name"] diff --git a/CMEW/app/configure_for/bin/test_fetch_recipe.py b/CMEW/app/configure_for/bin/test_fetch_recipe.py index 7829d111..74acd787 100755 --- a/CMEW/app/configure_for/bin/test_fetch_recipe.py +++ b/CMEW/app/configure_for/bin/test_fetch_recipe.py @@ -9,35 +9,28 @@ input for test_retrieve_specified, test_retrieve_defaults """ from fetch_recipe import retrieve_name_and_fp -from pathlib import Path -import pytest -@pytest.fixture -def mock_env_vars(monkeypatch): - # For adding extra datasets - monkeypatch.setenv( - "RECIPE_DICT_PATH", - str( - Path(__file__).parent.parent.parent - / "unittest" - / "mock_data" - / "recipe_paths.yml" - ), - ) +mock_recipe_dict = { + "mock_entry": { + "recipe_name": "recipe_specified_name.yml", + "recipe_fp": "subdir_1/recipe_second_name.yml", + "empty_additional_datasets": True, + }, +} -def test_retrieve_specified(mock_env_vars, monkeypatch): +def test_retrieve_specified(monkeypatch): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry") expected = "recipe_specified_name.yml", "subdir_1/recipe_second_name.yml" - actual = retrieve_name_and_fp() + actual = retrieve_name_and_fp(mock_recipe_dict) assert actual == expected -def test_retrieve_defaults(mock_env_vars, monkeypatch): +def test_retrieve_defaults(monkeypatch): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "not_here") expected = "recipe_not_here.yml", "recipe_not_here.yml" - actual = retrieve_name_and_fp() + actual = retrieve_name_and_fp(mock_recipe_dict) assert actual == expected diff --git a/CMEW/app/configure_for/bin/test_update_recipe_file.py b/CMEW/app/configure_for/bin/test_update_recipe_file.py index f27d648b..37e90d12 100644 --- a/CMEW/app/configure_for/bin/test_update_recipe_file.py +++ b/CMEW/app/configure_for/bin/test_update_recipe_file.py @@ -148,15 +148,13 @@ def test_remove_additional_datasets( path_to_recipe_additionals_removed, ): monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry") - monkeypatch.setenv( - "RECIPE_DICT_PATH", - str( - Path(__file__).parent.parent.parent - / "unittest" - / "mock_data" - / "recipe_paths.yml" - ), - ) + mock_recipe_dict = { + "mock_entry": { + "recipe_name": "recipe_specified_name.yml", + "recipe_fp": "subdir_1/recipe_second_name.yml", + "empty_additional_datasets": True, + }, + } with open(path_to_recipe_additionals_removed, "r") as file_handle_1: expected = yaml.safe_load(file_handle_1) @@ -165,7 +163,7 @@ def test_remove_additional_datasets( pre_recipe = yaml.safe_load(file_handle_2) # Using str(filepath) here as update_recipe_file.py uses os, not pathlib - actual = remove_additional_datasets(pre_recipe) + actual = remove_additional_datasets(pre_recipe, mock_recipe_dict) assert actual == expected diff --git a/CMEW/app/configure_for/bin/update_recipe_file.py b/CMEW/app/configure_for/bin/update_recipe_file.py index b2108555..12f92d73 100755 --- a/CMEW/app/configure_for/bin/update_recipe_file.py +++ b/CMEW/app/configure_for/bin/update_recipe_file.py @@ -11,6 +11,7 @@ import yaml import sys import logging +from config_configure_for import recipes_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) @@ -93,17 +94,21 @@ def add_extra_datasets(recipe, yaml_filepath): return recipe -def remove_additional_datasets(recipe): +def remove_additional_datasets(recipe, recipe_dict=recipes_dict): """ Optionally remove additional_datasets sections from an ESMValTool recipe. The option to remove additional datasets is controlled by the key - empty_additional_datasets in the YAML file at RECIPE_DICT_PATH. + empty_additional_datasets in the recipe_dict. Parameters ---------- recipe: dict The content of the recipe which may have additional datasets. + recipe_dict : dict + A dictionary with keys for a recipe identifier and the value + True assigned to an inner key of empty_additional_datasets + if additional datasets are to be emptied from a recipe. Returns ------- @@ -114,17 +119,12 @@ def remove_additional_datasets(recipe): # Look up the recipe and destination from the environment recipe_id = os.environ["CYLC_TASK_PARAM_recipe"] - # Load the yaml config file from ../etc - recipe_dict_fp = os.environ["RECIPE_DICT_PATH"] - logger.debug("Reading recipe dict from %s", recipe_dict_fp) - with open(recipe_dict_fp, "r") as f: - recipe_dict = yaml.safe_load(f) - logger.debug("Recipe dict:\n%s", recipe_dict) - # Don't empty by default empty_additionals = False - # Read specific recipe names and filepaths from the yaml config file + # Read specific recipe names and filepaths from the config file + logger.debug("Recipe dict:\n%s", recipe_dict) + if recipe_id in recipe_dict: logger.debug("Using info from recipe dictionary for %s", recipe_id) if "empty_additional_datasets" in recipe_dict[recipe_id]: diff --git a/CMEW/app/configure_for/etc/recipe_paths.yml b/CMEW/app/configure_for/etc/recipe_paths.yml deleted file mode 100644 index 26b3d03e..00000000 --- a/CMEW/app/configure_for/etc/recipe_paths.yml +++ /dev/null @@ -1,12 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -correlation: - recipe_name: recipe_correlation.yml - recipe_fp: examples/recipe_correlation.yml - empty_additional_datasets: true -python: - recipe_name: recipe_python.yml - recipe_fp: examples/recipe_python.yml -ref_cre: - recipe_name: recipe_ref_cre.yml - recipe_fp: ref/recipe_ref_cre.yml diff --git a/CMEW/app/configure_standardise/bin/config_configure_standardise.py b/CMEW/app/configure_standardise/bin/config_configure_standardise.py new file mode 100644 index 00000000..9bcd0455 --- /dev/null +++ b/CMEW/app/configure_standardise/bin/config_configure_standardise.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# (C) Crown Copyright 2026, Met Office. +# The LICENSE.md file contains full licensing details. + +# Information on which streams contain variables from different MIP tables. +streams_dict = { + "apm": [ + "Amon/hfls", + "Amon/hfss", + "Amon/rlds", + "Amon/rlut", + "Amon/rlutcs", + "Amon/rsds", + "Amon/rsdt", + "Amon/rsut", + "Amon/rsutcs", + "Amon/tas", + "Emon/rls", + "Emon/rss", + ], + "inm": [ + "SImon/siconc", + ], + "onm/grid-T": [ + "Omon/tos", + ], +} + +# Default information to write into the CDDS request file. +requests_defaults = { + "metadata": { + "base_date": "1850-01-01T00:00:00", + "branch_method": "no parent", + "license": ( + "GCModelDev model data is licensed under the " + "Open Government License v3 " + "(https://www.nationalarchives.gov.uk/" + "doc/open-government-licence/version/3/)" + ), + "mip": "ESMVal", + "mip_era": "GCModelDev", + "model_type": "AGCM AER", + }, + "common": { + "mode": "relaxed", + "package": "round-1", + }, + "data": { + "mass_data_class": "crum", + "model_workflow_branch": "trunk", + "model_workflow_revision": "not used except with data request", + }, + "misc": { + "atmos_timestep": 1200, + }, + "conversion": { + "mip_convert_plugin": "HadGEM3", + "skip_archive": True, + "cylc_args": "--no-detach -v", + }, +} diff --git a/CMEW/app/configure_standardise/bin/create_request_file.py b/CMEW/app/configure_standardise/bin/create_request_file.py index f4e9a3e1..cf7e8480 100755 --- a/CMEW/app/configure_standardise/bin/create_request_file.py +++ b/CMEW/app/configure_standardise/bin/create_request_file.py @@ -10,58 +10,35 @@ from pathlib import Path import yaml import logging +from config_configure_standardise import requests_defaults, streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) filename = os.path.basename(__file__) logger = logging.getLogger(filename) -def load_request_defaults(): +def list_streams(stream_dict=streams_dict): """ - Load default values for request file. + Lists only the streams in the stream_dict. - Returns - ------- - dict - CDDS request configuration default settings. - """ - # Get path to default settings - defaults = os.environ["REQUEST_DEFAULTS_PATH"] - - # Read the defaults - with open(defaults, "r") as f: - config = yaml.safe_load(f) - - logger.debug( - "Default config:\n%s", - config, - ) - return config - - -def list_streams(): - """ - Lists all streams in the ../etc/streams.yml file. + Parameters + ---------- + stream_dict : dict + A dictionary containing information about data streams. Returns ------- str Space separated list of all streams. """ - # Get path to stream mappings - streams_config = os.environ["STREAM_CONFIG_PATH"] - - # Read the stream mappings - with open(streams_config, "r") as f: - config = yaml.safe_load(f) - logger.debug( - "Stream config:\n%s", - config, - ) + # Load the stream information dictionary + logger.debug("Stream information:\n%s", stream_dict) # List all streams (keys) all_streams = [] - for stream in config: + for stream in stream_dict: + # For substreams we only want the first part + stream = stream.split("/")[0] all_streams.append(stream) # Return as a space separated list @@ -74,18 +51,25 @@ def list_streams(): return stream_str -def create_request(model_run): +def create_request(model_run, request_defaults=requests_defaults): """ Build a CDDS request configuration for a run identified by a suite_id. Uses information from the model_runs.yml file. + Parameters + ---------- + model_run : str + The suite ID as a model run identifier. + request_defaults : dict + A dictionary containing the CDDS request default values. + Returns ------- dict CDDS request configuration. """ - defaults = load_request_defaults() + defaults = request_defaults mip_table_dir = os.environ["MIP_TABLE_DIR"] @@ -103,8 +87,7 @@ def create_request(model_run): request = {} request["metadata"] = { **defaults["metadata"], - # The internal dictionary replaces the T with a space - "base_date": defaults["metadata"]["base_date"].isoformat(), + "base_date": defaults["metadata"]["base_date"], "calendar": dataset_dict["calendar"], "experiment_id": dataset_dict["experiment_id"], "institution_id": dataset_dict["institute"], diff --git a/CMEW/app/configure_standardise/bin/create_variables_file.py b/CMEW/app/configure_standardise/bin/create_variables_file.py index ad541139..3e5e9958 100755 --- a/CMEW/app/configure_standardise/bin/create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/create_variables_file.py @@ -5,9 +5,9 @@ Generates the variables.txt file from the ESMValTool recipe. """ import os -import yaml import sys import logging +from config_configure_standardise import streams_dict logging.basicConfig(level=logging.INFO, stream=sys.stdout) @@ -44,42 +44,21 @@ def combine_variable_lists(directory): return variables -def load_stream_dict(): - """ - Loads stream information from the ../etc/streams.yml file. - - Returns - ------- - dict - A mapping of pre-defined streams to their associated variables - """ - # Get path to stream mappings - streams_config = os.environ["STREAM_CONFIG_PATH"] - logger.debug("Reading streams from %s", streams_config) - - # Read the stream mappings - with open(streams_config, "r") as f: - config = yaml.safe_load(f) - - # Return the whole dictionary - return config - - -def add_stream_to_variables(variables): +def add_stream_to_variables(variables, stream_dict=streams_dict): """Add stream information to a list of variables. Parameters ---------- variables : list[str] List of variables in the format "MIP_table/variable_name" + stream_dict : dict + A dictionary containing information about data streams. Returns ------- list[str] List of variables in the format "MIP_table/variable_name:stream" """ - stream_dict = load_stream_dict() - # Using a second dictionary to avoid looping var_to_stream = { var: stream diff --git a/CMEW/app/configure_standardise/bin/test_create_request_file.py b/CMEW/app/configure_standardise/bin/test_create_request_file.py index 444e1035..4b293316 100644 --- a/CMEW/app/configure_standardise/bin/test_create_request_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_request_file.py @@ -9,7 +9,25 @@ """ from pathlib import Path import configparser -from create_request_file import create_request +from create_request_file import create_request, list_streams + + +def test_list_streams(): + mock_stream_dict = { + "abc": [ + "Amon/abcd", + "Aday/efgh", + ], + "def": [ + "SImon/ijkl", + ], + "xyz/grid-A": [ + "Omon/mnop", + ], + } + expected = "abc def xyz" + actual = list_streams(mock_stream_dict) + assert actual == expected def test_create_request(monkeypatch): @@ -17,11 +35,6 @@ def test_create_request(monkeypatch): "DATASETS_LIST_DIR", str(Path(__file__).parent.parent.parent / "unittest" / "mock_data"), ) - - request_defaults_path = ( - Path(__file__).parent.parent / "etc" / "request_defaults.yml" - ) - stream_config_path = Path(__file__).parent.parent / "etc" / "streams.yml" root_proc_dir = "/path/to/proc/dir/" root_data_dir = "/path/to/data/dir/" variables_path = "/path/to/variables.txt" @@ -29,15 +42,46 @@ def test_create_request(monkeypatch): stream_id = "apm inm" monkeypatch.setenv("RAW_DATA_DIR_MODE", "use_saved") - monkeypatch.setenv("REQUEST_DEFAULTS_PATH", str(request_defaults_path)) - monkeypatch.setenv("STREAM_CONFIG_PATH", str(stream_config_path)) monkeypatch.setenv("ROOT_PROC_DIR", root_proc_dir) monkeypatch.setenv("ROOT_DATA_DIR", root_data_dir) monkeypatch.setenv("VARIABLES_PATH", variables_path) monkeypatch.setenv("MIP_TABLE_DIR", mip_table_dir) monkeypatch.setenv("STREAM_ID", stream_id) - actual_request = create_request("u-cw673") + mock_request_defaults = { + "metadata": { + "base_date": "1850-01-01T00:00:00", + "branch_method": "no parent", + "license": ( + "GCModelDev model data is licensed under the " + "Open Government License v3 " + "(https://www.nationalarchives.gov.uk/" + "doc/open-government-licence/version/3/)" + ), + "mip": "ESMVal", + "mip_era": "GCModelDev", + "model_type": "AGCM AER", + }, + "common": { + "mode": "relaxed", + "package": "round-1", + }, + "data": { + "mass_data_class": "crum", + "model_workflow_branch": "trunk", + "model_workflow_revision": "not used except with data request", + }, + "misc": { + "atmos_timestep": 1200, + }, + "conversion": { + "mip_convert_plugin": "HadGEM3", + "skip_archive": True, + "cylc_args": "--no-detach -v", + }, + } + + actual_request = create_request("u-cw673", mock_request_defaults) cfg = configparser.ConfigParser() cfg.read_dict(actual_request) actual = {section: dict(cfg[section]) for section in cfg.sections()} diff --git a/CMEW/app/configure_standardise/bin/test_create_variables_file.py b/CMEW/app/configure_standardise/bin/test_create_variables_file.py index 1683ecd1..3666638b 100644 --- a/CMEW/app/configure_standardise/bin/test_create_variables_file.py +++ b/CMEW/app/configure_standardise/bin/test_create_variables_file.py @@ -57,9 +57,7 @@ def test_combine_variable_lists(): assert actual == expected -def test_add_stream_to_variables(monkeypatch, path_to_combined_variables): - stream_config_path = Path(__file__).parent.parent / "etc" / "streams.yml" - monkeypatch.setenv("STREAM_CONFIG_PATH", str(stream_config_path)) +def test_add_stream_to_variables(path_to_combined_variables): input = [ "Amon/hfls", "Amon/hfss", @@ -75,7 +73,27 @@ def test_add_stream_to_variables(monkeypatch, path_to_combined_variables): "Amon/tas", "SImon/siconc", ] - actual = add_stream_to_variables(input) + mock_stream_dict = { + "apm": [ + "Amon/hfls", + "Amon/hfss", + "Amon/rlds", + "Amon/rlut", + "Amon/rlutcs", + "Amon/rsds", + "Amon/rsdt", + "Amon/rsut", + "Amon/rsutcs", + "Amon/tas", + "Emon/rls", + "Emon/rss", + ], + "inm": [ + "SImon/siconc", + ], + } + + actual = add_stream_to_variables(input, mock_stream_dict) with open(path_to_combined_variables, "r") as file: expected = file.read().splitlines() diff --git a/CMEW/app/configure_standardise/etc/request_defaults.yml b/CMEW/app/configure_standardise/etc/request_defaults.yml deleted file mode 100644 index be203570..00000000 --- a/CMEW/app/configure_standardise/etc/request_defaults.yml +++ /dev/null @@ -1,22 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -metadata: - base_date: 1850-01-01T00:00:00 - branch_method: no parent - license: GCModelDev model data is licensed under the Open Government License v3 (https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) - mip: ESMVal - mip_era: GCModelDev - model_type: AGCM AER -common: - mode: relaxed - package: round-1 -data: - mass_data_class: crum - model_workflow_branch: trunk - model_workflow_revision: not used except with data request -misc: - atmos_timestep: 1200 -conversion: - mip_convert_plugin: HadGEM3 - skip_archive: True - cylc_args: --no-detach -v diff --git a/CMEW/app/configure_standardise/etc/streams.yml b/CMEW/app/configure_standardise/etc/streams.yml deleted file mode 100644 index 4fb1f357..00000000 --- a/CMEW/app/configure_standardise/etc/streams.yml +++ /dev/null @@ -1,17 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -apm: -- Amon/hfls -- Amon/hfss -- Amon/rlds -- Amon/rlut -- Amon/rlutcs -- Amon/rsds -- Amon/rsdt -- Amon/rsut -- Amon/rsutcs -- Amon/tas -- Emon/rls -- Emon/rss -inm: -- SImon/siconc diff --git a/CMEW/app/unittest/kgo/request_u-cw673.cfg b/CMEW/app/unittest/kgo/request_u-cw673.cfg index fa4ff4ee..ebf72891 100644 --- a/CMEW/app/unittest/kgo/request_u-cw673.cfg +++ b/CMEW/app/unittest/kgo/request_u-cw673.cfg @@ -26,7 +26,7 @@ model_workflow_revision = not used except with data request start_date = 1993-01-01T00:00:00 end_date = 2003-01-01T00:00:00 model_workflow_id = u-cw673 -streams = apm inm +streams = apm inm onm variable_list_file = /path/to/variables.txt [misc] diff --git a/CMEW/app/unittest/mock_data/recipe_paths.yml b/CMEW/app/unittest/mock_data/recipe_paths.yml deleted file mode 100644 index ff5b6b04..00000000 --- a/CMEW/app/unittest/mock_data/recipe_paths.yml +++ /dev/null @@ -1,6 +0,0 @@ -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -mock_entry: - recipe_name: recipe_specified_name.yml - recipe_fp: subdir_1/recipe_second_name.yml - empty_additional_datasets: true diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index ba946872..d57db030 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -73,7 +73,6 @@ [[RECIPE]] [[[environment]]] - RECIPE_DICT_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_for/etc/recipe_paths.yml RECIPE_PATH = "${CYLC_WORKFLOW_SHARE_DIR}/etc/recipe_${CYLC_TASK_PARAM_recipe}.yml" VARIABLES_LIST_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/variables_lists RECIPE_VARIABLES_PATH = ${VARIABLES_LIST_DIR}/${CYLC_TASK_PARAM_recipe}_variables.txt @@ -84,8 +83,6 @@ CDDS_SOFTWARE_DIR = "~cdds/software" RAW_DATA_DIR = {{ RAW_DATA_DIR | default("") }} RAW_DATA_DIR_MODE = {{ RAW_DATA_DIR_MODE }} - REQUEST_DEFAULTS_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_standardise/etc/request_defaults.yml - STREAM_CONFIG_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_standardise/etc/streams.yml # Workaround for bug in CDDS: ROOT_SOFTWARE_DIR: unbound variable. ROOT_SOFTWARE_DIR = ${CDDS_SOFTWARE_DIR} CDDS_VERSION = {{ CDDS_VERSION }}