Write files in etc directories as python files

CMEW/app/configure_for/bin/config_configure_for.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# (C) Crown Copyright 2026, Met Office.
+# The LICENSE.md file contains full licensing details.
+
+# Information about the locations of specific ESMValTool recipes.
+recipes_dict = {
+    "correlation": {
+        "recipe_name": "recipe_correlation.yml",
+        "recipe_fp": "examples/recipe_correlation.yml",
+        "empty_additional_datasets": True,
+    },
+    "python": {
+        "recipe_name": "recipe_python.yml",
+        "recipe_fp": "examples/recipe_python.yml",
+    },
+    "ref_cre": {
+        "recipe_name": "recipe_ref_cre.yml",
+        "recipe_fp": "ref/recipe_ref_cre.yml",
+    },
+}

CMEW/app/configure_for/bin/fetch_recipe.py

@@ -3,21 +3,29 @@
 # The LICENSE.md file contains full licensing details.
 import os
 import subprocess
-import yaml
 import sys
 import logging
+from config_configure_for import recipes_dict
+
 
 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 filename = os.path.basename(__file__)
 logger = logging.getLogger(filename)
 
 
-def retrieve_name_and_fp():
+def retrieve_name_and_fp(recipe_dict=recipes_dict):
     """
-    Looks in recipe_paths.yml for an entry or constructs default values.
+    Looks in the `recipe_dict` for an entry or constructs default values.
 
     Uses the environment variable CYLC_TASK_PARAM_recipe as a dict key.
 
+    Parameters
+    ----------
+    recipe_dict : dict
+        A dictionary with keys for recipe identifiers
+        for recipes which do not follow the default pattern
+        of names and locations within ESMValTool.
+
     Returns
     -------
     recipe_name: str
@@ -30,14 +38,8 @@ def retrieve_name_and_fp():
     recipe = os.environ["CYLC_TASK_PARAM_recipe"]
     logger.info("Fetching recipe %s", recipe)
 
-    # Load the yaml config file from ../etc
-    recipe_dict_fp = os.environ["RECIPE_DICT_PATH"]
-    logger.debug("Reading recipe dict from %s", recipe_dict_fp)
-    with open(recipe_dict_fp, "r") as f:
-        recipe_dict = yaml.safe_load(f)
+    # Read specific recipe names and filepaths from the config dict
     logger.debug("Recipe dict:\n%s", recipe_dict)
-
-    # Read specific recipe names and filepaths from the yaml config file
     if recipe in recipe_dict:
         logger.debug("Using info from recipe dictionary for %s", recipe)
         recipe_name = recipe_dict[recipe]["recipe_name"]

CMEW/app/configure_for/bin/test_fetch_recipe.py

@@ -9,35 +9,28 @@
     input for test_retrieve_specified, test_retrieve_defaults
 """
 from fetch_recipe import retrieve_name_and_fp
-from pathlib import Path
-import pytest
 
 
-@pytest.fixture
-def mock_env_vars(monkeypatch):
-    # For adding extra datasets
-    monkeypatch.setenv(
-        "RECIPE_DICT_PATH",
-        str(
-            Path(__file__).parent.parent.parent
-            / "unittest"
-            / "mock_data"
-            / "recipe_paths.yml"
-        ),
-    )
+mock_recipe_dict = {
+    "mock_entry": {
+        "recipe_name": "recipe_specified_name.yml",
+        "recipe_fp": "subdir_1/recipe_second_name.yml",
+        "empty_additional_datasets": True,
+    },
+}
 
 
-def test_retrieve_specified(mock_env_vars, monkeypatch):
+def test_retrieve_specified(monkeypatch):
     monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry")
     expected = "recipe_specified_name.yml", "subdir_1/recipe_second_name.yml"
-    actual = retrieve_name_and_fp()
+    actual = retrieve_name_and_fp(mock_recipe_dict)
 
     assert actual == expected
 
 
-def test_retrieve_defaults(mock_env_vars, monkeypatch):
+def test_retrieve_defaults(monkeypatch):
     monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "not_here")
     expected = "recipe_not_here.yml", "recipe_not_here.yml"
-    actual = retrieve_name_and_fp()
+    actual = retrieve_name_and_fp(mock_recipe_dict)
 
     assert actual == expected

CMEW/app/configure_for/bin/test_update_recipe_file.py

@@ -148,15 +148,13 @@ def test_remove_additional_datasets(
     path_to_recipe_additionals_removed,
 ):
     monkeypatch.setenv("CYLC_TASK_PARAM_recipe", "mock_entry")
-    monkeypatch.setenv(
-        "RECIPE_DICT_PATH",
-        str(
-            Path(__file__).parent.parent.parent
-            / "unittest"
-            / "mock_data"
-            / "recipe_paths.yml"
-        ),
-    )
+    mock_recipe_dict = {
+        "mock_entry": {
+            "recipe_name": "recipe_specified_name.yml",
+            "recipe_fp": "subdir_1/recipe_second_name.yml",
+            "empty_additional_datasets": True,
+        },
+    }
 
     with open(path_to_recipe_additionals_removed, "r") as file_handle_1:
         expected = yaml.safe_load(file_handle_1)
@@ -165,7 +163,7 @@ def test_remove_additional_datasets(
         pre_recipe = yaml.safe_load(file_handle_2)
 
     # Using str(filepath) here as update_recipe_file.py uses os, not pathlib
-    actual = remove_additional_datasets(pre_recipe)
+    actual = remove_additional_datasets(pre_recipe, mock_recipe_dict)
     assert actual == expected
 
 

CMEW/app/configure_for/bin/update_recipe_file.py

@@ -11,6 +11,7 @@
 import yaml
 import sys
 import logging
+from config_configure_for import recipes_dict
 
 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 filename = os.path.basename(__file__)
@@ -93,17 +94,21 @@ def add_extra_datasets(recipe, yaml_filepath):
     return recipe
 
 
-def remove_additional_datasets(recipe):
+def remove_additional_datasets(recipe, recipe_dict=recipes_dict):
     """
     Optionally remove additional_datasets sections from an ESMValTool recipe.
 
     The option to remove additional datasets is controlled by the key
-    empty_additional_datasets in the YAML file at RECIPE_DICT_PATH.
+    empty_additional_datasets in the recipe_dict.
 
     Parameters
     ----------
     recipe: dict
         The content of the recipe which may have additional datasets.
+    recipe_dict : dict
+        A dictionary with keys for a recipe identifier and the value
+        True assigned to an inner key of empty_additional_datasets
+        if additional datasets are to be emptied from a recipe.
 
     Returns
     -------
@@ -114,17 +119,12 @@ def remove_additional_datasets(recipe):
     # Look up the recipe and destination from the environment
     recipe_id = os.environ["CYLC_TASK_PARAM_recipe"]
 
-    # Load the yaml config file from ../etc
-    recipe_dict_fp = os.environ["RECIPE_DICT_PATH"]
-    logger.debug("Reading recipe dict from %s", recipe_dict_fp)
-    with open(recipe_dict_fp, "r") as f:
-        recipe_dict = yaml.safe_load(f)
-    logger.debug("Recipe dict:\n%s", recipe_dict)
-
     # Don't empty by default
     empty_additionals = False
 
-    # Read specific recipe names and filepaths from the yaml config file
+    # Read specific recipe names and filepaths from the config file
+    logger.debug("Recipe dict:\n%s", recipe_dict)
+
     if recipe_id in recipe_dict:
         logger.debug("Using info from recipe dictionary for %s", recipe_id)
         if "empty_additional_datasets" in recipe_dict[recipe_id]:

CMEW/app/configure_standardise/bin/config_configure_standardise.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+# (C) Crown Copyright 2026, Met Office.
+# The LICENSE.md file contains full licensing details.
+
+# Information on which streams contain variables from different MIP tables.
+streams_dict = {
+    "apm": [
+        "Amon/hfls",
+        "Amon/hfss",
+        "Amon/rlds",
+        "Amon/rlut",
+        "Amon/rlutcs",
+        "Amon/rsds",
+        "Amon/rsdt",
+        "Amon/rsut",
+        "Amon/rsutcs",
+        "Amon/tas",
+        "Emon/rls",
+        "Emon/rss",
+    ],
+    "inm": [
+        "SImon/siconc",
+    ],
+    "onm/grid-T": [
+        "Omon/tos",
+    ],
+}
+
+# Default information to write into the CDDS request file.
+requests_defaults = {
+    "metadata": {
+        "base_date": "1850-01-01T00:00:00",
+        "branch_method": "no parent",
+        "license": (
+            "GCModelDev model data is licensed under the "
+            "Open Government License v3 "
+            "(https://www.nationalarchives.gov.uk/"
+            "doc/open-government-licence/version/3/)"
+        ),
+        "mip": "ESMVal",
+        "mip_era": "GCModelDev",
+        "model_type": "AGCM AER",
+    },
+    "common": {
+        "mode": "relaxed",
+        "package": "round-1",
+    },
+    "data": {
+        "mass_data_class": "crum",
+        "model_workflow_branch": "trunk",
+        "model_workflow_revision": "not used except with data request",
+    },
+    "misc": {
+        "atmos_timestep": 1200,
+    },
+    "conversion": {
+        "mip_convert_plugin": "HadGEM3",
+        "skip_archive": True,
+        "cylc_args": "--no-detach -v",
+    },
+}

CMEW/app/configure_standardise/bin/create_request_file.py

@@ -10,58 +10,35 @@
 from pathlib import Path
 import yaml
 import logging
+from config_configure_standardise import requests_defaults, streams_dict
 
 logging.basicConfig(level=logging.INFO, stream=sys.stdout)
 filename = os.path.basename(__file__)
 logger = logging.getLogger(filename)
 
 
-def load_request_defaults():
+def list_streams(stream_dict=streams_dict):
     """
-    Load default values for request file.
+    Lists only the streams in the stream_dict.
 
-    Returns
-    -------
-    dict
-        CDDS request configuration default settings.
-    """
-    # Get path to default settings
-    defaults = os.environ["REQUEST_DEFAULTS_PATH"]
-
-    # Read the defaults
-    with open(defaults, "r") as f:
-        config = yaml.safe_load(f)
-
-    logger.debug(
-        "Default config:\n%s",
-        config,
-    )
-    return config
-
-
-def list_streams():
-    """
-    Lists all streams in the ../etc/streams.yml file.
+    Parameters
+    ----------
+    stream_dict : dict
+        A dictionary containing information about data streams.
 
     Returns
     -------
     str
         Space separated list of all streams.
     """
-    # Get path to stream mappings
-    streams_config = os.environ["STREAM_CONFIG_PATH"]
-
-    # Read the stream mappings
-    with open(streams_config, "r") as f:
-        config = yaml.safe_load(f)
-        logger.debug(
-            "Stream config:\n%s",
-            config,
-        )
+    # Load the stream information dictionary
+    logger.debug("Stream information:\n%s", stream_dict)
 
     # List all streams (keys)
     all_streams = []
-    for stream in config:
+    for stream in stream_dict:
+        # For substreams we only want the first part
+        stream = stream.split("/")[0]
         all_streams.append(stream)
 
     # Return as a space separated list
@@ -74,18 +51,25 @@ def list_streams():
     return stream_str
 
 
-def create_request(model_run):
+def create_request(model_run, request_defaults=requests_defaults):
     """
     Build a CDDS request configuration for a run identified by a suite_id.
 
     Uses information from the model_runs.yml file.
 
+    Parameters
+    ----------
+    model_run : str
+        The suite ID as a model run identifier.
+    request_defaults : dict
+        A dictionary containing the CDDS request default values.
+
     Returns
     -------
     dict
         CDDS request configuration.
     """
-    defaults = load_request_defaults()
+    defaults = request_defaults
 
     mip_table_dir = os.environ["MIP_TABLE_DIR"]
 
@@ -103,8 +87,7 @@ def create_request(model_run):
     request = {}
     request["metadata"] = {
         **defaults["metadata"],
-        # The internal dictionary replaces the T with a space
-        "base_date": defaults["metadata"]["base_date"].isoformat(),
+        "base_date": defaults["metadata"]["base_date"],
         "calendar": dataset_dict["calendar"],
         "experiment_id": dataset_dict["experiment_id"],
         "institution_id": dataset_dict["institute"],