Update the ozone diagnostics

.github/workflows/regression-mint.yaml

@@ -35,9 +35,11 @@ on:
         type: boolean
         default: false
 
-# Needed to commit the regenerated manifest/bundle back to the dispatched branch.
+# `contents` to commit the regenerated manifest/bundle back to the dispatched branch,
+# `pull-requests` to post the baseline diff on the branch's open pull request.
 permissions:
   contents: write
+  pull-requests: write
 
 # Never cancel an in-flight mint: it performs object-store writes and a push.
 concurrency:
@@ -67,6 +69,9 @@ jobs:
           exit 1
       - name: Check out repository
         uses: actions/checkout@v7
+        with:
+          # Full history so the baseline diff can find the merge-base with the default branch.
+          fetch-depth: 0
       - uses: ./.github/actions/setup
         with:
           python-version: "3.13"
@@ -109,6 +114,13 @@ jobs:
           if [ "${BUMP_VERSION}" = "true" ]; then args+=(--bump-version); fi
           if [ "${DRY_RUN}" = "true" ]; then args+=(--dry-run); fi
           uv run ref test-cases mint "${args[@]}"
+      - name: Fetch the default branch for the baseline diff
+        if: ${{ !inputs.dry_run }}
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          git fetch --no-tags origin \
+            "+refs/heads/${DEFAULT_BRANCH}:refs/remotes/origin/${DEFAULT_BRANCH}"
       - name: Commit regenerated baselines
         if: ${{ !inputs.dry_run }}
         env:
@@ -148,3 +160,37 @@ jobs:
           done
           echo "::error::failed to push regenerated baselines after retries"
           exit 1
+      - name: Report the baseline diff
+        # Runs after the commit so HEAD carries the regenerated manifests.
+        # Reporting only, so a failure here must not fail an otherwise successful mint.
+        if: ${{ !inputs.dry_run }}
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          # Two reports: the full one is small enough to keep as an artefact whatever the mint's
+          # size, while the capped one fits the comment and job-summary limits.
+          uv run python scripts/ci/mint_diff.py \
+            --base "origin/${DEFAULT_BRANCH}" \
+            --output mint-diff.md \
+            --comment-output mint-diff-comment.md
+
+          # The job summary always gets a report, whether or not a pull request exists.
+          cat mint-diff-comment.md >> "${GITHUB_STEP_SUMMARY}"
+
+          pr="$(gh pr list --head "${GITHUB_REF_NAME}" --state open --json number --jq '.[0].number')"
+          if [ -z "${pr}" ]; then
+            echo "No open pull request for ${GITHUB_REF_NAME}; the report is in the job summary only."
+            exit 0
+          fi
+          gh pr comment "${pr}" --body-file mint-diff-comment.md
+      - name: Upload the full baseline diff
+        # The comment is capped at GitHub's 65 KB limit, so keep the untruncated report too.
+        if: ${{ !inputs.dry_run }}
+        continue-on-error: true
+        uses: actions/upload-artifact@v4
+        with:
+          name: mint-diff
+          path: mint-diff.md
+          if-no-files-found: ignore

changelog/885.improvement.md

@@ -0,0 +1 @@
+Reduced the size of the ESMValTool conda environment.

changelog/886.fix.md

@@ -0,0 +1,4 @@
+Updates the ozone diagnostics to use the unified `ref/recipe_ref_ozone.yml` recipe from ESMValTool v2.15.
+This replaces the removed `recipe_ref_ozone_cmip7.yml`.
+The northern hemisphere polar cap now spans 60N to 85N, matching the upstream boundary fix.
+CMIP7 executions now evaluate the historical experiment through 2021.

changelog/887.fix.md

@@ -0,0 +1,2 @@
+Fabricated CMIP7 test data now reaches years without real CMIP6 source data by repeating the final year,
+rather than relabelling the whole series onto the requested end date.

packages/climate-ref-core/src/climate_ref_core/cmip6_to_cmip7.py

@@ -20,19 +20,17 @@
 import re
 import uuid
 from dataclasses import dataclass
-from datetime import UTC, datetime, timedelta
+from datetime import UTC, datetime
 from importlib import resources
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import attrs
 import cftime
 import numpy as np
 import pandas as pd
+import xarray as xr
 from loguru import logger
 
-if TYPE_CHECKING:
-    import xarray as xr
-
 
 def suppress_bounds_coordinates(ds: xr.Dataset) -> xr.Dataset:
     """
@@ -496,18 +494,17 @@ def _month_index(t: Any) -> int:
 _MONTHS_PER_YEAR = 12
 
 
-def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> xr.Dataset:
+def repeat_final_year_to(ds: xr.Dataset, end_year: int, end_month: int = 12) -> xr.Dataset:
     """
-    Relabel a monthly time axis so the final timestep lands on ``end_year``-``end_month``.
+    Extend a monthly series to ``end_year``-``end_month`` by repeating its final year.
 
-    The whole ``time`` coordinate (and any ``time_bnds``) is shifted by a constant
-    whole-month offset, so monthly spacing, calendar, and cftime types are preserved
-    and only the labels move. This is used to fabricate CMIP7 ``historical`` coverage
-    that reaches years for which no real data exists (e.g. 2002-2021 for the fire
-    diagnostic), without altering the underlying data values.
+    The real timesteps keep their real dates and values.
+    Any missing years are tiled from the last year.
+    This gives CMIP7 ``historical`` coverage for years the CMIP6 source never ran (e.g. 2015-2021)
+    without moving the years that did.
 
-    Only monthly data with a ``cftime`` time axis is supported; datasets without a
-    ``time`` coordinate (fixed-frequency, e.g. ``sftlf``) are returned unchanged.
+    Only monthly data with a ``cftime`` time axis is supported.
+    Datasets without a ``time`` coordinate (fixed-frequency, e.g. ``sftlf``) are returned unchanged.
 
     Parameters
     ----------
@@ -521,7 +518,15 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
     Returns
     -------
     xr.Dataset
-        A shallow copy with the relabelled time axis.
+        A dataset whose time axis runs to the requested end.
+
+    Raises
+    ------
+    TypeError
+        If the time axis is not ``cftime``.
+    ValueError
+        If ``end_month`` is outside 1-12, the series does not end in December,
+        or its final calendar year is incomplete.
     """
     if not 1 <= end_month <= _MONTHS_PER_YEAR:
         raise ValueError(f"end_month must be in 1..12, got {end_month}")
@@ -533,52 +538,53 @@ def shift_time_axis_end(ds: xr.Dataset, end_year: int, end_month: int = 12) -> x
     last = time_values[-1]
     if not isinstance(last, cftime.datetime):
         raise TypeError(
-            "shift_time_axis_end requires a cftime time axis; "
-            f"got {type(last).__name__}. Decode with use_cftime=True."
+            "repeat_final_year_to requires a cftime time axis. "
+            f"Got {type(last).__name__}. Decode with use_cftime=True."
         )
 
-    # Whole-month offset that moves the final label onto the requested end.
-    target_index = end_year * 12 + (end_month - 1)
-    offset_months = target_index - _month_index(last)
-    if offset_months == 0:
+    target_index = end_year * _MONTHS_PER_YEAR + (end_month - 1)
+    months_to_add = target_index - _month_index(last)
+    if months_to_add <= 0:
         return ds
 
-    calendar = last.calendar
-
-    def _days_in_month(year: int, month: int) -> int:
-        # Last day of ``month`` = day before the first of the following month,
-        # computed via cftime arithmetic so it respects the dataset's calendar
-        # (noleap, 360_day, etc.).
-        next_year, next_month = (year + 1, 1) if month == _MONTHS_PER_YEAR else (year, month + 1)
-        first_of_next = cftime.datetime(next_year, next_month, 1, calendar=calendar)
-        last_of_month = first_of_next - timedelta(days=1)  # type: ignore[operator]
-        return int(last_of_month.day)  # type: ignore[attr-defined]
-
-    def _shift(t: cftime.datetime) -> cftime.datetime:
-        total = _month_index(t) + offset_months
-        year, month = divmod(total, 12)
-        month += 1
-        # Clamp the day to the target month/calendar: a non-multiple-of-12 offset
-        # can land a day-31 (or leap Feb-29) label on a shorter month, which cftime
-        # would reject. Our monthly data is mid-month so this is normally a no-op.
-        day = min(t.day, _days_in_month(year, month))
-        return cftime.datetime(year, month, day, t.hour, t.minute, t.second, t.microsecond, calendar=calendar)
-
-    ds = ds.copy(deep=False)
-    shifted = np.array([_shift(t) for t in time_values])
-    new_time = ds["time"].copy(data=shifted)
-    new_time.encoding = dict(ds["time"].encoding)
-    ds = ds.assign_coords(time=new_time)
-
-    # Shift the matching time bounds, if present, so the axis stays self-consistent.
+    if last.month != _MONTHS_PER_YEAR:
+        raise ValueError(f"The series must end in December to repeat whole years, ends {last}")
+
     bounds_name = ds["time"].attrs.get("bounds")
-    if bounds_name and bounds_name in ds:
-        bnds_values = ds[bounds_name].values
-        shifted_bnds = np.array([[_shift(v) for v in row] for row in bnds_values])
-        ds[bounds_name] = ds[bounds_name].copy(data=shifted_bnds)
 
-    logger.debug(f"Shifted time axis by {offset_months} months so it ends {end_year:04d}-{end_month:02d}")
-    return ds
+    def _restamp(t: cftime.datetime, shift: int) -> cftime.datetime:
+        year = t.year + shift
+        try:
+            return t.replace(year=year)  # type: ignore[attr-defined,no-any-return]
+        except ValueError:
+            # A leap day has no counterpart in a common year, so take the day before.
+            return t.replace(year=year, day=t.day - 1)  # type: ignore[attr-defined,no-any-return]
+
+    def _relabel(block: xr.Dataset, shift: int) -> xr.Dataset:
+        relabelled = block.copy()
+        if bounds_name and bounds_name in relabelled:
+            bnds = [[_restamp(b, shift) for b in row] for row in block[bounds_name].values]
+            # Assign as (dims, data) so the new bounds are not aligned against the old time axis.
+            relabelled[bounds_name] = (block[bounds_name].dims, np.array(bnds))
+        return relabelled.assign_coords(time=[_restamp(t, shift) for t in block["time"].values])
+
+    final_year = ds.sel(time=str(last.year))
+    if len(final_year["time"]) != _MONTHS_PER_YEAR:
+        raise ValueError(f"A full final year is needed to repeat, got {len(final_year['time'])} months")
+
+    repeats = [_relabel(final_year, year - last.year) for year in range(last.year + 1, end_year + 1)]
+
+    extended = xr.concat([ds, *repeats], dim="time", data_vars="minimal", coords="minimal")
+    extended["time"].attrs = dict(ds["time"].attrs)
+    extended["time"].encoding = dict(ds["time"].encoding)
+
+    # The repeats are whole years, so drop anything past the requested month.
+    extended = extended.sel(time=slice(None, f"{end_year:04d}-{end_month:02d}"))
+
+    logger.debug(
+        f"Repeated {last.year} for {months_to_add} months so the series ends {end_year:04d}-{end_month:02d}"
+    )
+    return extended
 
 
 def convert_cmip6_dataset(
@@ -604,10 +610,10 @@ def convert_cmip6_dataset(
     inplace
         If True, modify the dataset in place; otherwise return a copy
     extend_historical_to
-        Opt-in ``(end_year, end_month)``. When set, the ``time`` axis is relabelled
-        via :func:`shift_time_axis_end` so the series ends on that month, fabricating
-        CMIP7 coverage for years without real data. Defaults to ``None`` (time axis
-        untouched), so existing conversions are byte-identical.
+        Opt-in ``(end_year, end_month)``. When set, the series is padded out to that month
+        via :func:`repeat_final_year_to`, fabricating CMIP7 coverage for years without real
+        data. Defaults to ``None`` (time axis untouched), so existing conversions are
+        byte-identical.
 
     Returns
     -------
@@ -619,7 +625,7 @@ def convert_cmip6_dataset(
 
     if extend_historical_to is not None:
         end_year, end_month = extend_historical_to
-        ds = shift_time_axis_end(ds, end_year=end_year, end_month=end_month)
+        ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
 
     # Determine the primary variable (skip coordinates/bounds)
     data_vars = [str(v) for v in ds.data_vars if not str(v).endswith("_bnds") and v not in ds.coords]

packages/climate-ref-core/src/climate_ref_core/esgf/cmip7.py

@@ -21,7 +21,7 @@
     format_cmip7_time_range,
     get_dreq_entry,
     get_frequency_from_table,
-    shift_time_axis_end,
+    repeat_final_year_to,
     suppress_bounds_coordinates,
 )
 from climate_ref_core.data import resolve_cache_dir
@@ -36,6 +36,37 @@ def _get_cmip7_cache_dir() -> Path:
     return cache_dir
 
 
+def _latest_file(files: list[Path]) -> Path | None:
+    """
+    Find the file holding a dataset's final timestep.
+
+    Only that file is extended, so a dataset split across several files keeps its earlier
+    chunks where they are instead of every chunk being padded out to the same end.
+
+    Parameters
+    ----------
+    files
+        The dataset's CMIP6 source files. Files without a time axis are skipped.
+
+    Returns
+    -------
+    Path | None
+        The file ending last, or ``None`` when none of them carry a time axis.
+    """
+    time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
+
+    latest: Path | None = None
+    latest_end = None
+    for path in files:
+        with xr.open_dataset(path, decode_times=time_coder) as ds:
+            if "time" not in ds.coords or len(ds["time"]) == 0:
+                continue
+            end = ds["time"].values[-1]
+            if latest_end is None or end > latest_end:
+                latest, latest_end = path, end
+    return latest
+
+
 def _convert_file_to_cmip7(
     cmip6_path: Path,
     cmip7_facets: dict[str, Any],
@@ -51,8 +82,8 @@ def _convert_file_to_cmip7(
     cmip7_facets
         CMIP7 facets for the output path
     extend_historical_to
-        Opt-in ``(end_year, end_month)`` passed to :func:`convert_cmip6_dataset`
-        to relabel the time axis so historical coverage reaches that month.
+        Opt-in ``(end_year, end_month)``.
+        The series is padded out to that month by repeating its final year, so historical coverage reaches it.
         Defaults to ``None`` (time axis untouched).
 
     Returns
@@ -85,12 +116,12 @@ def _convert_file_to_cmip7(
 
     time_coder = xr.coders.CFDatetimeCoder(use_cftime=True)
     with xr.open_dataset(cmip6_path, decode_times=time_coder) as ds:
-        # When fabricating extended historical coverage, relabel the time axis first
-        # so both the filename time range and the written data reflect the new dates.
+        # When fabricating extended historical coverage, pad the series first so both the
+        # filename time range and the written data reflect the added months.
         source_ds = ds
         if extend_historical_to is not None:
             end_year, end_month = extend_historical_to
-            source_ds = shift_time_axis_end(ds, end_year=end_year, end_month=end_month)
+            source_ds = repeat_final_year_to(ds, end_year=end_year, end_month=end_month)
 
         frequency = str(cmip7_facets.get("frequency", "mon"))
         time_range = format_cmip7_time_range(source_ds, frequency)
@@ -199,11 +230,11 @@ def __init__(
         time_span
             Optional time range filter (start, end) in YYYY-MM format
         extend_historical_to
-            Opt-in ``(end_year, end_month)``. When set, each converted CMIP7 file has
-            its time axis relabelled so historical coverage ends on that month, letting
-            us fabricate CMIP7 data for years without real CMIP6 source data (e.g. the
-            fire diagnostic's 2002-2021 window). Defaults to ``None`` (time axis
-            untouched), so other CMIP7 conversions are unchanged.
+            Opt-in ``(end_year, end_month)``. When set, a converted CMIP7 file that stops
+            short has its final year repeated until it ends on that month, letting us
+            fabricate CMIP7 data for years without real CMIP6 source data (e.g. the fire
+            diagnostic's 2002-2021 window). Files that already reach it are untouched, as
+            are all conversions when this defaults to ``None``.
         """
         self.slug = slug
         self.facets = facets
@@ -303,13 +334,19 @@ def fetch_datasets(self) -> pd.DataFrame:
 
             # Get file paths and convert them
             files = row_dict.get("files", [])
+            latest = None
+            if self.extend_historical_to is not None:
+                latest = _latest_file([Path(f) for f in files if Path(f).exists()])
+
             converted_files = []
             for file_path in files:
                 cmip6_path = Path(file_path)
                 if cmip6_path.exists():
                     try:
                         cmip7_path = _convert_file_to_cmip7(
-                            cmip6_path, cmip7_row, extend_historical_to=self.extend_historical_to
+                            cmip6_path,
+                            cmip7_row,
+                            extend_historical_to=self.extend_historical_to if cmip6_path == latest else None,
                         )
                         converted_files.append(str(cmip7_path))
                     except Exception as e: