From fdb5e7eeacf4e2c2c91244b69ee6494c789f9dca Mon Sep 17 00:00:00 2001 From: "eva.tapai" Date: Thu, 12 Nov 2026 09:47:58 +0100 Subject: [PATCH] Run tests on PRs to make sure recipes always have mocked and live tests as well --- .github/workflows/recipe-tests.yml | 19 +++++++++++++++ tests/test_every_recipe_has_a_test.py | 33 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/workflows/recipe-tests.yml create mode 100644 tests/test_every_recipe_has_a_test.py diff --git a/.github/workflows/recipe-tests.yml b/.github/workflows/recipe-tests.yml new file mode 100644 index 0000000..3184c1f --- /dev/null +++ b/.github/workflows/recipe-tests.yml @@ -0,0 +1,19 @@ +name: Recipe tests + +on: + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - run: pip install -r requirements.txt + # Includes the check that every recipe has a mocked and a live test. + - name: Run tests + run: python -m pytest diff --git a/tests/test_every_recipe_has_a_test.py b/tests/test_every_recipe_has_a_test.py new file mode 100644 index 0000000..19c4dc0 --- /dev/null +++ b/tests/test_every_recipe_has_a_test.py @@ -0,0 +1,33 @@ +from pathlib import Path + +from recipes import discover + +TESTS = Path(__file__).parent +MOCKED_TESTS = TESTS / "test_recipes.py" +LIVE_TESTS = TESTS / "test_live.py" + + +def untested_recipes(test_file, reference): + """Recipes missing from tests.""" + source = test_file.read_text() + return [r["id"] for r in discover() if reference.format(id=r["id"]) not in source] + + +def test_every_recipe_has_a_mocked_test(): + """We need mocked tests for each recipe because we build the current list of APIs from them.""" + untested = untested_recipes(MOCKED_TESTS, "recipes.{id}") + assert not untested, ( + f"Recipe(s) without a mocked test in tests/test_recipes.py: {', '.join(untested)}. " + "Add a test that runs the recipe against mocked Canvas responses, then run " + "`python scripts/list_apis.py` to update used-apis.txt." + ) + + +def test_every_recipe_has_a_live_test(): + """Live tests catch API changes the mocks can't. The PR author runs them: pytest -m live.""" + untested = untested_recipes(LIVE_TESTS, 'recipe("{id}")') + assert not untested, ( + f"Recipe(s) without a live test in tests/test_live.py: {', '.join(untested)}. " + "Add a test that runs the recipe against a real demo Canvas, and make sure " + "`pytest -m live` passes before you open the PR." + )