feat: add scaffolding to migrate subpackages to the new layout

.github/scripts/discover_subpackages.py

@@ -0,0 +1,198 @@
+"""Discover which subpackages a PR touches; output a CI matrix.
+
+Implements the PR-scoping rules and the stacked-PR override (when
+``base_ref`` is not ``main``, the matrix is promoted to all packages so
+the stacked branch sees the full effect of its parent's changes).
+
+Used by ``.github/workflows/ci.yml``'s ``discover`` job.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+DOCS_PREFIXES = ("docs/", "RFCs/")
+DOCS_FILE_SUFFIX = (".md",)
+COOKIECUTTER_PREFIX = "cookiecutter/"
+ROOT_TRIGGER_FILES = {"pyproject.toml", "uv.lock"}
+WORKFLOWS_PREFIX = ".github/workflows/"
+SHARED_SCRIPTS_PREFIX = ".github/scripts/"
+LEGACY_PREFIX = "mellea_contribs/"
+
+
+@dataclass
+class DiscoverInputs:
+    """Inputs to :func:`discover`."""
+
+    changed_files: list[str]
+    base_ref: str
+    repo_root: Path
+
+
+@dataclass
+class DiscoverResult:
+    """Result of :func:`discover`."""
+
+    matrix: list[str] = field(default_factory=list)
+    run_template_smoke: bool = False
+    reason: str = ""
+
+
+def _is_docs_only(files: list[str]) -> bool:
+    return (
+        all(f.startswith(DOCS_PREFIXES) or f.endswith(DOCS_FILE_SUFFIX) for f in files)
+        and len(files) > 0
+    )
+
+
+def _has_cookiecutter_change(files: list[str]) -> bool:
+    return any(f.startswith(COOKIECUTTER_PREFIX) for f in files)
+
+
+def _has_root_trigger(files: list[str]) -> bool:
+    return any(
+        f in ROOT_TRIGGER_FILES
+        or f.startswith(WORKFLOWS_PREFIX)
+        or f.startswith(SHARED_SCRIPTS_PREFIX)
+        for f in files
+    )
+
+
+def _changed_subpackages(files: list[str], all_subpackages: list[str]) -> list[str]:
+    """Among the new-flat subpackages, which ones did this PR touch?"""
+    touched: set[str] = set()
+    sub_set = set(all_subpackages)
+    for f in files:
+        first_segment = f.split("/", 1)[0] if "/" in f else f
+        if first_segment in sub_set:
+            touched.add(first_segment)
+    return sorted(touched)
+
+
+def discover(
+    inputs: DiscoverInputs, all_subpackages: list[str] | None = None
+) -> DiscoverResult:
+    """Compute which subpackages need CI for the given changed-files set."""
+    if all_subpackages is None:
+        all_subpackages = []
+    files = inputs.changed_files
+
+    # Stacked-PR override: when base_ref isn't main, run all packages so the
+    # stacked branch sees the full effect of its parent's changes.
+    if inputs.base_ref and inputs.base_ref != "main":
+        return DiscoverResult(
+            matrix=list(all_subpackages),
+            run_template_smoke=False,
+            reason="stacked-PR (base_ref != main -> all-packages)",
+        )
+
+    if not files:
+        return DiscoverResult(matrix=[], run_template_smoke=False, reason="no-changes")
+
+    # Docs-only short-circuit.
+    if _is_docs_only(files):
+        return DiscoverResult(matrix=[], run_template_smoke=False, reason="docs-only")
+
+    template_smoke = _has_cookiecutter_change(files)
+
+    # Root-level / workflow / shared-script trigger: run everything.
+    if _has_root_trigger(files):
+        return DiscoverResult(
+            matrix=list(all_subpackages),
+            run_template_smoke=template_smoke,
+            reason="root-or-workflow-change (all-packages)",
+        )
+
+    # Subpackage-scoped changes.
+    touched = _changed_subpackages(files, all_subpackages)
+
+    # If only legacy changes hit, the new ci.yml runs nothing (legacy-ci.yml
+    # picks up these PRs).
+    legacy_only = (
+        all(
+            f.startswith(LEGACY_PREFIX) or f.startswith(COOKIECUTTER_PREFIX)
+            for f in files
+        )
+        and not touched
+    )
+    if legacy_only and not template_smoke:
+        return DiscoverResult(
+            matrix=[], run_template_smoke=False, reason="legacy-only-no-new-ci"
+        )
+
+    if touched:
+        reason = "subpackage-scoped"
+    elif template_smoke:
+        reason = "cookiecutter-only"
+    else:
+        reason = "no-trigger"
+
+    return DiscoverResult(
+        matrix=touched, run_template_smoke=template_smoke, reason=reason
+    )
+
+
+def _load_subpackages_from_repo(repo_root: Path) -> list[str]:
+    """Find new-flat subpackages: top-level dirs containing a pyproject.toml."""
+    subs: list[str] = []
+    exclude = {"cookiecutter", "mellea_contribs", ".github", ".venv", "RFCs", "docs"}
+    for child in sorted(repo_root.iterdir()):
+        if not child.is_dir() or child.name in exclude or child.name.startswith("."):
+            continue
+        if (child / "pyproject.toml").exists():
+            subs.append(child.name)
+    return subs
+
+
+def main() -> int:
+    """CLI entrypoint."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--changed-files",
+        required=True,
+        help="Newline-separated list of changed files",
+    )
+    parser.add_argument("--base-ref", required=True)
+    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
+    args = parser.parse_args()
+
+    files = [f.strip() for f in args.changed_files.splitlines() if f.strip()]
+    inputs = DiscoverInputs(
+        changed_files=files,
+        base_ref=args.base_ref,
+        repo_root=args.repo_root,
+    )
+    all_subs = _load_subpackages_from_repo(args.repo_root)
+    result = discover(inputs, all_subpackages=all_subs)
+
+    print(
+        f"Discover: matrix={result.matrix}, "
+        f"template_smoke={result.run_template_smoke}, reason={result.reason}",
+        file=sys.stderr,
+    )
+
+    output = {
+        "matrix": result.matrix,
+        "run_template_smoke": result.run_template_smoke,
+        "reason": result.reason,
+    }
+    gh_out = os.environ.get("GITHUB_OUTPUT")
+    if gh_out:
+        with open(gh_out, "a") as f:
+            f.write(f"matrix={json.dumps(result.matrix)}\n")
+            f.write(
+                "run_template_smoke="
+                f"{'true' if result.run_template_smoke else 'false'}\n"
+            )
+            f.write(f"reason={result.reason}\n")
+    print(json.dumps(output))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/grandfather_legacy.json

@@ -0,0 +1,12 @@
+{
+  "_comment": "Legacy old-flat subpackages under mellea_contribs/ that pre-date the restructure. Each entry is removed by the corresponding migration PR. After migration is complete, this list is empty and the file is deleted.",
+  "legacy_paths": [
+    "mellea_contribs/__init__.py",
+    "mellea_contribs/dspy_backend",
+    "mellea_contribs/crewai_backend",
+    "mellea_contribs/langchain_backend",
+    "mellea_contribs/mellea-integration-core",
+    "mellea_contribs/reqlib_package",
+    "mellea_contribs/tools_package"
+  ]
+}

.github/scripts/open_per_package_bump_prs.py

@@ -0,0 +1,180 @@
+"""Open one bump PR per subpackage in dependency-tier order.
+
+Used by ``.github/workflows/receive-mellea-release.yml`` after every
+mellea release. Replaces the "one big PR" model with one PR per
+subpackage so owners can merge their bump independently — slow movers
+don't block fast movers, and per-package CI signals compatibility.
+
+Tiers (subpackages within a tier are opened concurrently; tiers serialize):
+
+- Tier 1: ``_integration_core`` (consumed by frameworks).
+- Tier 2: framework subpackages — ``dspy``, ``langchain``, ``crewai``,
+  ``tools``. Opened once Tier 1's PR has merged.
+- Tier 3: leaf subpackages with no contribs-internal dependencies —
+  ``legal-reqs``, ``python-imports``, ``grounding-context``. Opened once
+  Tier 2 has merged.
+
+The script runs once per workflow invocation and opens only the PRs for
+the *current* tier — i.e., the lowest tier with subpackages that still
+need a bump and whose dependencies are already on the target version.
+The workflow re-runs (via the bot itself observing tier-N PRs being
+merged) to advance to the next tier.
+
+Subpackages whose ``pyproject.toml`` doesn't exist or doesn't need a
+bump (already at target version) are skipped silently.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from update_contribs_versions import UnacceptableMelleaLine, update_pyproject
+
+# Tier definitions — see module docstring. The tier name is used as a
+# stable label only; the receiver looks up subpackages by directory name.
+TIERS: list[tuple[str, list[str]]] = [
+    ("integration-core", ["_integration_core", "mellea-integration-core"]),
+    ("frameworks", ["dspy", "langchain", "crewai", "tools"]),
+    ("leaves", ["legal-reqs", "python-imports", "grounding-context"]),
+]
+
+
+def _subpackage_needs_bump(pyproject: Path, target: str) -> bool:
+    """True iff editing pyproject would actually change [project] version."""
+    if not pyproject.exists():
+        return False
+    text = pyproject.read_text()
+    # Cheap signal: target version string already present in a [project]
+    # block? Not perfect but avoids reparsing.
+    needle = f'version = "{target}"'
+    return needle not in text
+
+
+def _detect_tier(repo: Path, target: str) -> tuple[str, list[Path]] | None:
+    """Return (tier_name, list_of_subpackage_pyprojects_that_need_bump).
+
+    Returns the lowest tier that has at least one subpackage needing a bump.
+    If every tier is already on target, returns None.
+    """
+    for tier_name, candidates in TIERS:
+        needs_bump: list[Path] = []
+        for sub in candidates:
+            pyproject = repo / sub / "pyproject.toml"
+            if _subpackage_needs_bump(pyproject, target):
+                needs_bump.append(pyproject)
+        if needs_bump:
+            return tier_name, needs_bump
+    return None
+
+
+def _open_pr(repo: Path, subpackage: Path, target: str) -> None:
+    """Branch, edit, lock, commit, push, and gh-pr-create for one subpackage.
+
+    Idempotent at the branch level: if the branch already exists upstream,
+    skip cleanly (the previous run already opened this PR).
+    """
+    sub_dir = subpackage.parent
+    name = sub_dir.name
+    branch = f"sync-mellea-{target}-{name}"
+
+    # Skip if branch already exists upstream (PR already opened).
+    result = subprocess.run(
+        ["git", "ls-remote", "--exit-code", "origin", f"refs/heads/{branch}"],
+        cwd=repo,
+        capture_output=True,
+    )
+    if result.returncode == 0:
+        print(f"  [{name}] branch {branch} already exists upstream — skipping.")
+        return
+
+    # Bump pyproject.toml.
+    try:
+        update_pyproject(subpackage, target)
+    except UnacceptableMelleaLine as exc:
+        print(f"  [{name}] error: {exc}", file=sys.stderr)
+        sys.exit(2)
+
+    # Refresh uv.lock.
+    subprocess.run(
+        ["uv", "lock", "--upgrade-package", "mellea"],
+        cwd=sub_dir,
+        check=True,
+    )
+
+    # Branch, commit, push.
+    subprocess.run(["git", "checkout", "-b", branch], cwd=repo, check=True)
+    subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
+    subprocess.run(
+        ["git", "commit", "-m", f"chore({name}): bump version to v{target}"],
+        cwd=repo,
+        check=True,
+    )
+    subprocess.run(["git", "push", "origin", branch], cwd=repo, check=True)
+
+    # Open PR.
+    body = (
+        f"Automated bump of `{name}`'s `[project] version` to `{target}` "
+        f"after the matching mellea release.\n\n"
+        f"`mellea>=` constraint untouched — owners control that floor.\n\n"
+        f"Merging this PR is independent of any other tier-{name} subpackage's "
+        f"bump PR; please verify CI passes."
+    )
+    subprocess.run(
+        [
+            "gh", "pr", "create",
+            "--title", f"chore({name}): bump version to v{target}",
+            "--body", body,
+            "--head", branch,
+            "--base", "main",
+        ],
+        cwd=repo,
+        check=True,
+    )
+
+    # Reset to main for the next iteration.
+    subprocess.run(["git", "checkout", "main"], cwd=repo, check=True)
+    subprocess.run(["git", "reset", "--hard", "origin/main"], cwd=repo, check=True)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("repo_root", type=Path, help="contribs repo root")
+    parser.add_argument("version", help="target version (matches mellea release)")
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="report which tier and subpackages would be bumped, but don't edit",
+    )
+    args = parser.parse_args()
+
+    if not args.repo_root.is_dir():
+        print(f"error: {args.repo_root} is not a directory", file=sys.stderr)
+        return 1
+
+    detected = _detect_tier(args.repo_root, args.version)
+    if detected is None:
+        print(f"All subpackages already at v{args.version}; nothing to bump.")
+        return 0
+
+    tier_name, pyprojects = detected
+    print(f"Active tier: {tier_name} ({len(pyprojects)} subpackage(s) need bumping)")
+    for pp in pyprojects:
+        print(f"  - {pp.parent.name}")
+
+    if args.dry_run:
+        # Emit JSON for the workflow to consume.
+        print(json.dumps({"tier": tier_name, "subpackages": [pp.parent.name for pp in pyprojects]}))
+        return 0
+
+    for pp in pyprojects:
+        _open_pr(args.repo_root, pp, args.version)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())