Backport PR #3607: (chore): derive CI matrix from hatch env

Author: flying-sheep

.github/workflows/ci.yml

@@ -17,43 +17,49 @@ defaults:
     shell: bash -e {0} # -e to fail on error
 
 jobs:
-  pytest:
+  get-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      envs: ${{ steps.get-envs.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          filter: blob:none
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: false
+      - id: get-envs
+        run: |
+          ENVS_JSON=$(NO_COLOR=1 uvx hatch env show --json | jq -c 'to_entries
+            | map(
+              select(.key | startswith("hatch-test"))
+              | {
+                name: .key,
+                "test-type": (if (.key | test("pre|min")) then "coverage" else null end),
+                python: .value.python,
+              }
+            )')
+          echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT
+
+  test:
+    needs: get-environments
     runs-on: ubuntu-latest
-
     strategy:
       matrix:
-        include:
-          - python-version: '3.10'
-          - python-version: '3.12'
-          - python-version: '3.12'
-            dependencies-version: min-optional
-          - python-version: '3.12'
-            dependencies-version: pre-release
-            test-type: coverage
-          - python-version: '3.10'
-            dependencies-version: minimum
-            test-type: coverage
-
-    env:  # for use codecov’s env_vars tagging
-      PYTHON: ${{ matrix.python-version }}
-      DEPS: ${{ matrix.dependencies-version || 'default' }}
-      TESTS: ${{ matrix.test-type || 'default' }}
-
+        env: ${{ fromJSON(needs.get-environments.outputs.envs) }}
+    env:  # environment variable for use in codecov’s env_vars tagging
+      ENV_NAME: ${{ matrix.env.name }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
           filter: blob:none
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install UV
-        uses: astral-sh/setup-uv@v5
+      - uses: astral-sh/setup-uv@v5
         with:
           enable-cache: true
+          python-version: ${{ matrix.env.python }}
           cache-dependency-glob: pyproject.toml
 
       - name: Cache downloaded data
@@ -63,55 +69,42 @@ jobs:
           key: pytest
 
       - name: Install dependencies
-        if: matrix.dependencies-version == null
-        run: uv pip install --system --compile "scanpy[dev,test-full] @ ."
-      - name: Install dependencies (no optional features)
-        if: matrix.dependencies-version == 'min-optional'
-        run: uv pip install --system --compile "scanpy[dev,test-min] @ ."
-      - name: Install dependencies (minimum versions)
-        if: matrix.dependencies-version == 'minimum'
-        run: |
-          uv pip install --system --compile tomli packaging
-          deps=$(python3 ci/scripts/min-deps.py pyproject.toml --extra dev test)
-          uv pip install --system --compile $deps "scanpy @ ."
-      - name: Install dependencies (pre-release versions)
-        if: matrix.dependencies-version == 'pre-release'
-        run: uv pip install -v --system --compile --pre "scanpy[dev,test-full] @ ." "anndata[dev,test] @ git+https://github.com/scverse/anndata.git"
+        run: uvx hatch -v env create ${{ matrix.env.name }}
 
-      - name: Run pytest
-        if: matrix.test-type == null
-        run: pytest
-      - name: Run pytest (coverage)
-        if: matrix.test-type == 'coverage'
-        run: coverage run -m pytest --cov --cov-report=xml
+      - name: Run tests
+        if: matrix.env.test-type == null
+        run: uvx hatch run ${{ matrix.env.name }}:run
+      - name: Run tests (coverage)
+        if: matrix.env.test-type == 'coverage'
+        run: uvx hatch run ${{ matrix.env.name }}:run-cov --cov --cov-report=xml
 
       - name: Upload coverage data
-        uses: codecov/codecov-action@v4
-        if: matrix.test-type == 'coverage'
+        uses: codecov/codecov-action@v5
+        if: matrix.env.test-type == 'coverage'
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          env_vars: "PYTHON,DEPS,TESTS"
+          env_vars: ENV_NAME
           fail_ci_if_error: true
-          file: test-data/coverage.xml
+          files: test-data/coverage.xml
 
       - name: Upload test results
         # yaml strings can’t start with “!”, so using explicit substitution
         if: ${{ !cancelled() }}
         uses: codecov/test-results-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          env_vars: "PYTHON,DEPS,TESTS"
+          env_vars: ENV_NAME
           fail_ci_if_error: true
           file: test-data/test-results.xml
 
       - name: Publish debug artifacts
-        if: matrix.test-type == 'coverage'
+        if: matrix.env.test-type == 'coverage'
         uses: actions/upload-artifact@v4
         with:
-          name: debug-data-${{ matrix.python-version }}-${{ matrix.dependencies-version || 'default' }}-${{ matrix.test-type || 'default' }}
+          name: debug-data-${{ matrix.env.name }}
           path: .pytest_cache/d/debug
 
-  check-build:
+  build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -126,3 +119,15 @@ jobs:
           enable-cache: false
       - run: uvx --from build pyproject-build --sdist --wheel .
       - run: uvx twine check dist/*
+
+  check:
+    if: always()
+    needs:
+      - get-environments
+      - test
+      - build
+    runs-on: ubuntu-latest
+    steps:
+        - uses: re-actors/alls-green@release/v1
+          with:
+            jobs: ${{ toJSON(needs) }}

.gitignore

@@ -27,7 +27,7 @@
 
 # Python build files
 __pycache__/
-/ci/scanpy-min-deps.txt
+/ci/scanpy-low-vers.txt
 /dist/
 /*-env/
 /env-*/

ci/scripts/min-deps.py ci/scripts/low-vers.pyci/scripts/min-deps.py renamed to ci/scripts/low-vers.py

@@ -5,7 +5,7 @@
 #   "packaging",
 # ]
 # ///
-"""Parse a pyproject.toml file and output a list of minimum dependencies."""
+"""Parse a pyproject.toml file and output a list of minimum dependency versions."""
 
 from __future__ import annotations
 
@@ -37,8 +37,9 @@ def min_dep(req: Requirement) -> Requirement:
     Example
     -------
     >>> min_dep(Requirement("numpy>=1.0"))
-    <Requirement('numpy==1.0.*')>
-
+    <Requirement('numpy==1.0')>
+    >>> min_dep(Requirement("numpy<3.0"))
+    <Requirement('numpy<3.0')>
     """
     req_name = req.name
     if req.extras:
@@ -48,25 +49,26 @@ def min_dep(req: Requirement) -> Requirement:
         spec for spec in req.specifier if spec.operator in {"==", "~=", ">=", ">"}
     ]
     if not filter_specs:
-        return Requirement(req_name)
-
+        # TODO: handle markers
+        return Requirement(f"{req_name}{req.specifier}")
     min_version = Version("0.0.0.a1")
     for spec in filter_specs:
         if spec.operator in {">", ">=", "~="}:
             min_version = max(min_version, Version(spec.version))
         elif spec.operator == "==":
             min_version = Version(spec.version)
 
-    return Requirement(f"{req_name}=={min_version}.*")
+    return Requirement(f"{req_name}=={min_version}")
 
 
 def extract_min_deps(
     dependencies: Iterable[Requirement], *, pyproject
 ) -> Generator[Requirement, None, None]:
-    """Extract minimum dependencies from a list of requirements."""
+    """Extract minimum dependency versions from a list of requirements."""
     dependencies = deque(dependencies)  # We'll be mutating this
     project_name = pyproject["project"]["name"]
 
+    deps = {}
     while len(dependencies) > 0:
         req = dependencies.pop()
 
@@ -79,7 +81,11 @@ def extract_min_deps(
                 extra_deps = pyproject["project"]["optional-dependencies"][extra]
                 dependencies += map(Requirement, extra_deps)
         else:
-            yield min_dep(req)
+            if req.name in deps:
+                req.specifier &= deps[req.name].specifier
+                req.extras |= deps[req.name].extras
+            deps[req.name] = min_dep(req)
+    yield from deps.values()
 
 
 class Args(argparse.Namespace):
@@ -105,6 +111,7 @@ def parser(cls) -> argparse.ArgumentParser:
             prog="min-deps",
             description=cls.__doc__,
             usage="pip install `python min-deps.py pyproject.toml`",
+            allow_abbrev=False,
         )
         parser.add_argument(
             "_path",

hatch.toml

@@ -15,25 +15,27 @@ scripts.clean = "git restore --source=HEAD --staged --worktree -- docs/release-n
 
 [envs.hatch-test]
 default-args = [  ]
-features = [ "dev", "test", "dask-ml" ]
+features = [ "dev", "test-min" ]
 extra-dependencies = [ "ipykernel" ]
 overrides.matrix.deps.env-vars = [
     { if = [ "pre" ], key = "UV_PRERELEASE", value = "allow" },
-    { if = [ "min" ], key = "UV_CONSTRAINT", value = "ci/scanpy-min-deps.txt" },
+    { if = [ "low-vers" ], key = "UV_CONSTRAINT", value = "ci/scanpy-low-vers.txt" },
 ]
 overrides.matrix.deps.pre-install-commands = [
-    { if = [ "min" ], value = "uv run ci/scripts/min-deps.py pyproject.toml --all-extras -o ci/scanpy-min-deps.txt" },
+    { if = [
+        "low-vers",
+    ], value = "uv run ci/scripts/low-vers.py pyproject.toml --all-extras -o ci/scanpy-low-vers.txt" },
 ]
 overrides.matrix.deps.python = [
-    { if = [ "min" ], value = "3.10" },
-    { if = [ "stable", "full", "pre" ], value = "3.12" },
-]
-overrides.matrix.deps.features = [
-    { if = [ "full" ], value = "test-full" },
+    { if = [ "low-vers" ], value = "3.10" },
+    { if = [ "stable", "pre" ], value = "3.12" },
 ]
 overrides.matrix.deps.extra-dependencies = [
     { if = [ "pre" ], value = "anndata[dev,test] @ git+https://github.com/scverse/anndata.git" },
 ]
+overrides.matrix.deps.features = [
+    { if = [ "stable", "pre", "low-vers" ], value = "test" },
+]
 
 [[envs.hatch-test.matrix]]
-deps = [ "stable", "full", "pre", "min" ]
+deps = [ "stable", "pre", "low-vers", "few-extras" ]

pyproject.toml

@@ -54,18 +54,18 @@ dependencies = [
     "seaborn>=0.13",
     "h5py>=3.7",
     "tqdm",
-    "scikit-learn>=1.1",
-    "statsmodels>=0.14",
+    "scikit-learn>=1.1.3",
+    "statsmodels>=0.14.4",
     "patsy!=1.0.0",                               # https://github.com/pydata/patsy/issues/215
     "networkx>=2.7",
     "natsort",
     "joblib",
     "numba>=0.57",
-    "umap-learn>=0.5,!=0.5.0",
+    "umap-learn>=0.5.13",
     "pynndescent>=0.5",
     "packaging>=21.3",
     "session-info2",
-    "legacy-api-wrap>=1.4",                       # for positional API deprecations
+    "legacy-api-wrap>=1.4.1",                     # for positional API deprecations
     "typing-extensions; python_version < '3.13'",
 ]
 dynamic = [ "version" ]
@@ -94,22 +94,15 @@ test-min = [
 ]
 test = [
     "scanpy[test-min]",
-    # Optional but important dependencies
-    "scanpy[leiden]",
-    "zarr<3",
+    # optional storage and processing modes
     "scanpy[dask]",
-    "scanpy[scrublet]",
-]
-test-full = [
-    "scanpy[test]",
-    # optional storage modes
     "zappy",
+    "zarr<3",
     # additional tested algorithms
+    "scanpy[scrublet]",
     "scanpy[louvain]",
-    "scanpy[magic]",
+    "scanpy[leiden]",
     "scanpy[skmisc]",
-    "scanpy[harmony]",
-    "scanpy[scanorama]",
     "scanpy[dask-ml]",
 ]
 doc = [
@@ -139,14 +132,14 @@ dev = [
 ]
 # Algorithms
 paga = [ "igraph" ]
-louvain = [ "igraph", "louvain>=0.6.0,!=0.6.2" ] # Louvain community detection
-leiden = [ "igraph>=0.10", "leidenalg>=0.9.0" ]  # Leiden community detection
-bbknn = [ "bbknn" ]                              # Batch balanced KNN (batch correction)
-magic = [ "magic-impute>=2.0" ]                  # MAGIC imputation method
-skmisc = [ "scikit-misc>=0.1.3" ]                # highly_variable_genes method 'seurat_v3'
-harmony = [ "harmonypy" ]                        # Harmony dataset integration
-scanorama = [ "scanorama" ]                      # Scanorama dataset integration
-scrublet = [ "scikit-image" ]                    # Doublet detection with automatic thresholds
+louvain = [ "igraph", "louvain>=0.6.0,!=0.6.2" ]  # Louvain community detection
+leiden = [ "igraph>=0.10.8", "leidenalg>=0.9.0" ] # Leiden community detection
+bbknn = [ "bbknn" ]                               # Batch balanced KNN (batch correction)
+magic = [ "magic-impute>=2.0.4" ]                 # MAGIC imputation method
+skmisc = [ "scikit-misc>=0.1.3" ]                 # highly_variable_genes method 'seurat_v3'
+harmony = [ "harmonypy" ]                         # Harmony dataset integration
+scanorama = [ "scanorama" ]                       # Scanorama dataset integration
+scrublet = [ "scikit-image" ]                     # Doublet detection with automatic thresholds
 # Acceleration
 rapids = [ "cudf>=0.9", "cuml>=0.9", "cugraph>=0.9" ] # GPU accelerated calculation of neighbors
 dask = [ "dask[array]>=2022.09.2" ]                   # Use the Dask parallelization engine