From 65cba971b9f2c956427203ed53dabf7224b44981 Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Fri, 21 Aug 2026 11:54:31 +0100 Subject: [PATCH 1/6] #514: Remove fetch_recipe.py --- CMEW/app/configure_for/bin/command_line.py | 65 -------------- CMEW/app/configure_for/bin/fetch_recipe | 8 -- CMEW/app/configure_for/bin/fetch_recipe.py | 89 ------------------- .../configure_for/bin/test_fetch_recipe.py | 22 ----- CMEW/app/configure_for/rose-app.conf | 7 +- CMEW/flow.cylc | 4 +- 6 files changed, 6 insertions(+), 189 deletions(-) delete mode 100755 CMEW/app/configure_for/bin/fetch_recipe delete mode 100644 CMEW/app/configure_for/bin/fetch_recipe.py delete mode 100755 CMEW/app/configure_for/bin/test_fetch_recipe.py diff --git a/CMEW/app/configure_for/bin/command_line.py b/CMEW/app/configure_for/bin/command_line.py index 718d20ea..5341a9e6 100644 --- a/CMEW/app/configure_for/bin/command_line.py +++ b/CMEW/app/configure_for/bin/command_line.py @@ -2,7 +2,6 @@ # The LICENSE.md file contains full licensing details. import argparse from get_variables_from_recipe import get_variables_from_recipe -from fetch_recipe import fetch_recipe from update_recipe_file import update_recipe_file @@ -58,70 +57,6 @@ def main_for_get_variables_from_recipe(arguments=None): get_variables_from_recipe(args.recipe_path, args.output_filepath) -def parse_args_for_fetch_recipe(arguments): - """ - Return the names and values of the command line arguments for - :func:`main_for_fetch_recipe`. - - Parameters - ---------- - arguments : :obj:`list` of :obj:`str` - The command line arguments to be parsed. - - Returns - ------- - :class:`argparse.Namespace` - The names and values of the command line arguments. - """ - parser = argparse.ArgumentParser( - description="Retrieve an ESMValTool recipe.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - "--recipe_id", - help=( - "The short identifier of the recipe to retrieve, " - "as written in the `flow.cylc` file." - ), - ) - parser.add_argument( - "--recipe_dict_fp", - help=( - "The filepath of the YAML file containing the " - "name and location within esmvaltool.recipes " - "of the recipe to be fetched." - ), - ) - parser.add_argument( - "--output_filepath", - help=( - "The full path to the where the " - "ESMValTool recipe will be written." - ), - ) - return parser.parse_args(arguments) - - -def main_for_fetch_recipe(arguments=None): - """ - Retrieve an ESMValTool recipe. - - Parameters - ---------- - arguments : :obj:`list` of :obj:`str` - The command line arguments to be parsed. - """ - # Parse the arguments. - args = parse_args_for_fetch_recipe(arguments) - - # Run the code. - print("Fetching recipe.") - print(f"Recipe ID: {args.recipe_id}") - print(f"Recipe dict filepath: {args.recipe_dict_fp}") - print(f"Output filepath: {args.output_filepath}") - fetch_recipe(args.recipe_id, args.recipe_dict_fp, args.output_filepath) - - def parse_args_for_update_recipe_file(arguments): """ Return the names and values of the command line arguments for diff --git a/CMEW/app/configure_for/bin/fetch_recipe b/CMEW/app/configure_for/bin/fetch_recipe deleted file mode 100755 index 1e202e59..00000000 --- a/CMEW/app/configure_for/bin/fetch_recipe +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -from command_line import main_for_fetch_recipe - - -if __name__ == "__main__": - main_for_fetch_recipe() diff --git a/CMEW/app/configure_for/bin/fetch_recipe.py b/CMEW/app/configure_for/bin/fetch_recipe.py deleted file mode 100644 index 46cfd3cc..00000000 --- a/CMEW/app/configure_for/bin/fetch_recipe.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -import os -import subprocess -import yaml -import sys -import logging - -logging.basicConfig(level=logging.INFO, stream=sys.stdout) -filename = os.path.basename(__file__) -logger = logging.getLogger(filename) - - -def retrieve_name_and_fp(recipe_id, recipe_dict_fp): - """ - Return the name of a recipe and its location within - ``esmvaltool/recipes``. - - Parameters - ---------- - recipe_id : str - The short identifier of the recipe to retrieve, - as written in the `flow.cylc` file. - recipe_dict_fp : str - The filepath of the YAML file containing the name and location - within esmvaltool.recipes of the recipe to be fetched. - - Returns - ------- - recipe_name: str - The name of the recipe, usually of the form recipe_thing.yml - recipe_internal_loc: str - The location of the recipe within esmvaltool/recipes - - """ - logger.info("Fetching recipe %s", recipe_id) - - # Load the yaml config file from ../etc - with open(recipe_dict_fp, "r") as f: - recipe_dict = yaml.safe_load(f) - logger.debug("Recipe dict:\n%s", recipe_dict) - - # Read specific recipe names and filepaths from the yaml config file - if recipe_id in recipe_dict: - logger.debug("Using info from recipe dictionary for %s", recipe_id) - recipe_name = recipe_dict[recipe_id]["recipe_name"] - recipe_internal_loc = recipe_dict[recipe_id]["recipe_fp"] - - # Or use the defaults - else: - logger.debug( - "Using default name and filepath for recipe %s", recipe_id - ) - recipe_name = f"recipe_{recipe_id}.yml" - recipe_internal_loc = recipe_name - - return recipe_name, recipe_internal_loc - - -def fetch_recipe(recipe_id, recipe_dict_fp, output_filepath): - """ - Fetch a recipe from ESMValTool and copy it to the output filepath. - - Parameters - ---------- - recipe_id : str - The short identifier of the recipe to retrieve, - as written in the `flow.cylc` file. - recipe_dict_fp : str - The filepath of the YAML file containing the name and location - within esmvaltool.recipes of the recipe to be fetched. - output_filepath: str - The full path to the where the ESMValTool recipe will be written. - """ - # Find the full name and location within ESMValTool - recipe_name, recipe_internal_loc = retrieve_name_and_fp( - recipe_id, recipe_dict_fp - ) - - # Build the command to fetch and move the recipe - command = f""" - esmvaltool recipes get {recipe_internal_loc} - mv {recipe_name} {output_filepath} - """ - - # Run the command - logging.info("Running command: %s", command) - subprocess.run(command, shell=True) diff --git a/CMEW/app/configure_for/bin/test_fetch_recipe.py b/CMEW/app/configure_for/bin/test_fetch_recipe.py deleted file mode 100755 index f6bed8ae..00000000 --- a/CMEW/app/configure_for/bin/test_fetch_recipe.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# (C) Crown Copyright 2026, Met Office. -# The LICENSE.md file contains full licensing details. -"""Unit tests for fetch_recipe.py.""" -from fetch_recipe import retrieve_name_and_fp -from configure_for_conftest import recipe_paths_yml_fp - - -def test_retrieve_specified(): - test_recipe_id = "mock_entry" - expected = "recipe_specified_name.yml", "subdir_1/recipe_second_name.yml" - actual = retrieve_name_and_fp(test_recipe_id, str(recipe_paths_yml_fp())) - - assert actual == expected - - -def test_retrieve_defaults(): - test_recipe_id = "not_here" - expected = "recipe_not_here.yml", "recipe_not_here.yml" - actual = retrieve_name_and_fp(test_recipe_id, str(recipe_paths_yml_fp())) - - assert actual == expected diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index 7b00e21a..c1a71f2b 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -3,10 +3,9 @@ [command] default=set -euo pipefail - =cmew-esmvaltool-env fetch_recipe \ - =--recipe_id $CYLC_TASK_PARAM_recipe \ - =--recipe_dict_fp $RECIPE_DICT_PATH \ - =--output_filepath $RECIPE_PATH + =ESMVALTOOL_RECIPE_PATH=$(cmew-esmvaltool-env esmvaltool recipes list | grep $CYLC_TASK_PARAM_recipe) + =cmew-esmvaltool-env esmvaltool recipes get $ESMVALTOOL_RECIPE_PATH + =mv $RECIPE_NAME $RECIPE_PATH =cmew-esmvaltool-env update_recipe_file \ =--recipe_path $RECIPE_PATH \ =--model_runs_yml_fp ${DATASETS_LIST_DIR}/model_runs.yml \ diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index 1a5339f5..86d31723 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -77,7 +77,9 @@ [[RECIPE]] [[[environment]]] RECIPE_DICT_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_for/etc/recipe_paths.yml - RECIPE_PATH = "${CYLC_WORKFLOW_SHARE_DIR}/etc/recipe_${CYLC_TASK_PARAM_recipe}.yml" + RECIPE_DIR = "${CYLC_WORKFLOW_SHARE_DIR}/etc" + RECIPE_NAME = "recipe_${CYLC_TASK_PARAM_recipe}.yml" + RECIPE_PATH = "${RECIPE_DIR}/${RECIPE_NAME}" VARIABLES_LIST_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/variables_lists RECIPE_VARIABLES_PATH = ${VARIABLES_LIST_DIR}/${CYLC_TASK_PARAM_recipe}_variables.txt From 73e5f2e6e37cc98f02d8173ee993395a9942dd82 Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Fri, 21 Aug 2026 12:03:32 +0100 Subject: [PATCH 2/6] #514: Remove unused variable --- CMEW/flow.cylc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMEW/flow.cylc b/CMEW/flow.cylc index 86d31723..f6cb04d8 100644 --- a/CMEW/flow.cylc +++ b/CMEW/flow.cylc @@ -77,9 +77,8 @@ [[RECIPE]] [[[environment]]] RECIPE_DICT_PATH = ${CYLC_WORKFLOW_RUN_DIR}/app/configure_for/etc/recipe_paths.yml - RECIPE_DIR = "${CYLC_WORKFLOW_SHARE_DIR}/etc" RECIPE_NAME = "recipe_${CYLC_TASK_PARAM_recipe}.yml" - RECIPE_PATH = "${RECIPE_DIR}/${RECIPE_NAME}" + RECIPE_PATH = "${CYLC_WORKFLOW_SHARE_DIR}/etc/${RECIPE_NAME}" VARIABLES_LIST_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/variables_lists RECIPE_VARIABLES_PATH = ${VARIABLES_LIST_DIR}/${CYLC_TASK_PARAM_recipe}_variables.txt From c651402e50108fbb6781edbb5895ffc193980145 Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Mon, 24 Aug 2026 15:27:36 +0100 Subject: [PATCH 3/6] Update CMEW/app/configure_for/rose-app.conf Co-authored-by: Naomi Parsons --- CMEW/app/configure_for/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index c1a71f2b..8af4d353 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -3,7 +3,7 @@ [command] default=set -euo pipefail - =ESMVALTOOL_RECIPE_PATH=$(cmew-esmvaltool-env esmvaltool recipes list | grep $CYLC_TASK_PARAM_recipe) + =ESMVALTOOL_RECIPE_PATH=$(cmew-esmvaltool-env esmvaltool recipes list | grep $CYLC_TASK_PARAM_recipe | tail -n 1) =cmew-esmvaltool-env esmvaltool recipes get $ESMVALTOOL_RECIPE_PATH =mv $RECIPE_NAME $RECIPE_PATH =cmew-esmvaltool-env update_recipe_file \ From ae4e120dc12aab482d20e6ed688799cff88ebd47 Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Mon, 24 Aug 2026 15:39:07 +0100 Subject: [PATCH 4/6] #514: Add a comment to explain the reason for using 'tail' --- CMEW/app/configure_for/rose-app.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index 8af4d353..b8fe8c18 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -3,6 +3,9 @@ [command] default=set -euo pipefail + # Select only the last match to avoid selecting any log messages + # from 'cmew-esmvaltool-env' that may contain the recipe name, + # e.g. from the 'run_name'. =ESMVALTOOL_RECIPE_PATH=$(cmew-esmvaltool-env esmvaltool recipes list | grep $CYLC_TASK_PARAM_recipe | tail -n 1) =cmew-esmvaltool-env esmvaltool recipes get $ESMVALTOOL_RECIPE_PATH =mv $RECIPE_NAME $RECIPE_PATH From e3adde854c5c65dacb4f6c27988a5bb2e9a2a34a Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Mon, 24 Aug 2026 15:43:35 +0100 Subject: [PATCH 5/6] #514: Move the comment outside the default statement --- CMEW/app/configure_for/rose-app.conf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index b8fe8c18..5ff8481a 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -1,11 +1,12 @@ # (C) Crown Copyright 2024-2026, Met Office. # The LICENSE.md file contains full licensing details. +# Select only the last match to avoid selecting any log messages +# from 'cmew-esmvaltool-env' that may contain the recipe name, +# e.g. from the 'run_name'. + [command] default=set -euo pipefail - # Select only the last match to avoid selecting any log messages - # from 'cmew-esmvaltool-env' that may contain the recipe name, - # e.g. from the 'run_name'. =ESMVALTOOL_RECIPE_PATH=$(cmew-esmvaltool-env esmvaltool recipes list | grep $CYLC_TASK_PARAM_recipe | tail -n 1) =cmew-esmvaltool-env esmvaltool recipes get $ESMVALTOOL_RECIPE_PATH =mv $RECIPE_NAME $RECIPE_PATH From c69b570aff09fd858d3caac69577b94fe288bcba Mon Sep 17 00:00:00 2001 From: Emma Hogan Date: Mon, 24 Aug 2026 15:45:37 +0100 Subject: [PATCH 6/6] #514: Make it part of the first comment --- CMEW/app/configure_for/rose-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMEW/app/configure_for/rose-app.conf b/CMEW/app/configure_for/rose-app.conf index 5ff8481a..802ff686 100644 --- a/CMEW/app/configure_for/rose-app.conf +++ b/CMEW/app/configure_for/rose-app.conf @@ -1,6 +1,6 @@ # (C) Crown Copyright 2024-2026, Met Office. # The LICENSE.md file contains full licensing details. - +# # Select only the last match to avoid selecting any log messages # from 'cmew-esmvaltool-env' that may contain the recipe name, # e.g. from the 'run_name'.