diff --git a/checks/default.nix b/checks/default.nix index b1d81180..f4072624 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -17,6 +17,7 @@ in nixbot-effects-tests = self.packages.${pkgs.stdenv.hostPlatform.system}.nixbot-effects.tests.pytest; effects-lib-ssh = import ./effects-lib-ssh.nix checkArgs; effects-lib-run-nixos = import ./effects-lib-run-nixos.nix checkArgs; + effects-lib-flake-update = import ./effects-lib-flake-update.nix checkArgs; sqlc-generated = import ./sqlc.nix checkArgs; effects-lib = import ./effects-lib.nix checkArgs; docs-examples = import ./docs-examples.nix checkArgs; diff --git a/checks/docs-examples.nix b/checks/docs-examples.nix index c1ccb179..77ea7bfb 100644 --- a/checks/docs-examples.nix +++ b/checks/docs-examples.nix @@ -74,6 +74,7 @@ let primaryRepo = tag: { rev = "0123456789abcdef"; + forgeType = "github"; branch = if tag == null then "main" else null; inherit tag; }; diff --git a/checks/effects-lib-flake-update.nix b/checks/effects-lib-flake-update.nix new file mode 100644 index 00000000..f82682ee --- /dev/null +++ b/checks/effects-lib-flake-update.nix @@ -0,0 +1,432 @@ +# effects-lib's `flakeUpdate`: its setup and effect scripts run in a +# checkout set up as nixbot sets one up, with the checkout's token in +# origin and in an insteadOf for the forge. The forge is local bare +# repositories behind an http URL (a global insteadOf that only applies +# once the effect has dropped the checkout's) and a fake API at it. The +# stand-in nix rewrites flake.lock and commits it like +# `nix flake update --commit-lock-file`. +{ pkgs, ... }: +let + inherit (pkgs) lib; + effects = import ../herculesCI/effects-lib.nix { inherit pkgs; }; + fakeNix = pkgs.writeShellScriptBin "nix" '' + printf '%s\n' "$*" >>"$TMPDIR/nix-args" + # Stable: the same update every run, as when no input moved. + if [[ -n ''${FAKE_NIX_STABLE:-} ]]; then lock="$*"; else lock=$RANDOM$RANDOM; fi + echo "{ \"run\": \"$lock\" }" >flake.lock + git commit -q -m "flake.lock: Update" -m "Updated input 'nixpkgs'" flake.lock + ''; + fakeForge = pkgs.writers.writePython3 "fake-forge" { } '' + import http.server + import json + import re + import sys + + # The forges' replies to a second pull request for the same branch. + EXISTS = { + "/api/v3/repos/acme/{}/pulls": (422, { + "message": "Validation Failed", + "errors": [{"message": "A pull request already exists for " + "acme:flake-update."}], + }), + "/api/v1/repos/acme/{}/pulls": (409, { + "message": "pull request already exists for these targets", + }), + "/api/v4/projects/acme%2F{}/merge_requests": (409, { + "message": ["Another open merge request already exists for " + "this source branch: !1"], + }), + } + seen = set() + + + class Forge(http.server.BaseHTTPRequestHandler): + def reply(self, status, body): + self.send_response(status) + self.end_headers() + self.wfile.write(json.dumps(body).encode()) + + def record(self): + length = int(self.headers["Content-Length"] or 0) + body = json.loads(self.rfile.read(length)) if length else None + with open(sys.argv[2], "a") as log: + entry = {"method": self.command, "path": self.path, + "body": body, "auth": self.headers["Authorization"]} + print(json.dumps(entry, sort_keys=True, separators=(",", ":")), + file=log) + return body + + def do_POST(self): + body = self.record() + if self.path == "/api/graphql": + # The "clean" repository's pull request is mergeable now. + if body["variables"]["id"] == "PR_clean": + return self.reply(200, {"errors": [{ + "message": "Pull request is in clean status"}]}) + return self.reply(200, {"data": {}}) + if self.path.endswith("/merge"): + return self.reply(200, {}) + repo = re.search(r"acme(?:/|%2F)([^/]+)", self.path)[1] + template = self.path.replace(repo, "{}", 1) + pr = (self.path, body.get("head") or body["source_branch"]) + if pr in seen: + return self.reply(*EXISTS[template]) + seen.add(pr) + self.reply(201, {"number": 1, "node_id": f"PR_{repo}"}) + + def do_GET(self): + self.record() + path = self.path.split("?", 1)[0] + repo = re.search(r"acme/([^/]+)", path)[1] + self.reply(200, [ + {"number": 1, "node_id": f"PR_{repo}", "head": {"ref": head}} + for p, head in seen if p == path + ]) + + def do_PUT(self): + self.record() + self.reply(200, {"merged": True}) + + def log_message(self, *args): + pass + + + server = http.server.HTTPServer(("127.0.0.1", 0), Forge) + with open(sys.argv[1], "w") as f: + f.write(str(server.server_port)) + server.serve_forever() + ''; + # Two root inputs, which perInput updates one by one. + seedLock = builtins.toJSON { + nodes.root.inputs = { + nixpkgs = "nixpkgs"; + utils = "utils"; + }; + root = "root"; + version = 7; + }; + effectFor = + forgeType: args: + effects.flakeUpdate ( + { + inherit forgeType; + nix = fakeNix; + baseMergeMethod = "reset"; + pullRequestTitle = "chore: update flake.lock"; + } + // args + ); + hook = + lib.findFirst (p: lib.getName p == "hercules-ci-effect-sh") (throw "no setup hook") + (effects.mkEffect { }).nativeBuildInputs; + run = + { + forge, + repo ? "widget", + effect, + # Without a baseMergeBranch the update branch is updated as it stands. + resets ? true, + branches ? [ "flake-update" ], + # Each run updates the same: the second must not push. + stable ? false, + check ? "", + expected, + }: + '' + remote=$TMPDIR/forge/acme/${repo}.git + rm -rf "$TMPDIR/forge" "$TMPDIR/checkout" "$TMPDIR/requests" "$HOME" + mkdir -p "$HOME" + git init -q --bare -b develop "$remote" + git clone -q "$remote" "$TMPDIR/seed" 2>/dev/null + (cd "$TMPDIR/seed" && echo ${lib.escapeShellArg seedLock} >flake.lock && git add flake.lock \ + && git commit -qm init && git push -q origin HEAD:develop) + rm -rf "$TMPDIR/seed" + develop=$(git -C "$remote" rev-parse develop) + + # The forge's git side. git takes the longest matching insteadOf, so + # while the checkout's (one character longer) is there, URLs go to + # the fake API with its token and fail. + forge="http://127.0.0.1:$port" + git config --global url."$TMPDIR/forge".insteadOf "$forge" + # nixbot's checkout: its token in origin and in every forge URL. + git clone -q "$remote" "$TMPDIR/checkout" + git -C "$TMPDIR/checkout" remote set-url origin \ + "http://x-access-token:checkout-token@127.0.0.1:$port/acme/${repo}.git" + # Two values under one key, as nixbot sets for https and ssh. + for prefix in "$forge/" "git@127.0.0.1:"; do + git -C "$TMPDIR/checkout" config --add \ + url."http://x-access-token:checkout-token@127.0.0.1:$port/".insteadOf "$prefix" + done + + for attempt in 1 2; do + heads=$(git -C "$remote" for-each-ref --format='%(objectname)' refs/heads) + ( + ${lib.optionalString stable "export FAKE_NIX_STABLE=1"} + cd "$TMPDIR/checkout" + export PATH=${lib.makeBinPath effect.nativeBuildInputs}:$PATH + export NIXBOT_EFFECT_CHECKOUT=$TMPDIR/checkout + set -euo pipefail + eval ${lib.escapeShellArg effect.userSetupScript} + eval ${lib.escapeShellArg effect.effectScript} + ) + done + + # Fetched and pushed with the effect's token, not the checkout's. + grep -qx "http://${ + if forge == "gitlab" then "oauth2" else "x-access-token" + }:secret-token@127.0.0.1:$port" ~/.git-credentials + ${lib.optionalString (forge == "github") '' + grep -qx "access-tokens = 127.0.0.1:$port=secret-token" ~/.config/nix/nix.conf + ''} + ${lib.optionalString stable '' + [[ $(git -C "$remote" for-each-ref --format='%(objectname)' refs/heads) == "$heads" ]] + ''} + ${check} + # One update per branch and run. + [[ $(grep -c . "$TMPDIR/nix-args") == ${toString (2 * builtins.length branches)} ]] + rm "$TMPDIR/nix-args" + for branch in ${lib.escapeShellArgs branches}; do + ${ + if resets then + '' + # Reset: each run starts again from develop and force-pushes. + [[ $(git -C "$remote" rev-parse "$branch^") == "$develop" ]] + '' + else + '' + # No base merge: the second run updates the branch as it stands. + [[ $(git -C "$remote" rev-parse "$branch~2") == "$develop" ]] + '' + } + done + diff "$TMPDIR/requests" ${pkgs.writeText "${forge}-${repo}-requests" expected} + ''; + create = + forge: repo: + { + github = { + method = "POST"; + auth = "Bearer secret-token"; + body = { + base = "develop"; + body = "Updated input 'nixpkgs'"; + head = "flake-update"; + title = "chore: update flake.lock"; + }; + path = "/api/v3/repos/acme/${repo}/pulls"; + }; + gitea = { + method = "POST"; + auth = "token secret-token"; + body = { + base = "develop"; + body = "Updated input 'nixpkgs'"; + head = "flake-update"; + title = "chore: update flake.lock"; + }; + path = "/api/v1/repos/acme/${repo}/pulls"; + }; + gitlab = { + method = "POST"; + auth = "Bearer secret-token"; + body = { + description = "Updated input 'nixpkgs'"; + source_branch = "flake-update"; + target_branch = "develop"; + title = "chore: update flake.lock"; + }; + path = "/api/v4/projects/acme%2F${repo}/merge_requests"; + }; + } + .${forge}; + autoMerge = repo: method: { + method = "POST"; + auth = "Bearer secret-token"; + body = { + query = "mutation ($id: ID!, $method: PullRequestMergeMethod!) { enablePullRequestAutoMerge(input: {pullRequestId: $id, mergeMethod: $method}) { clientMutationId } }"; + variables = { + id = "PR_${repo}"; + method = lib.toUpper method; + }; + }; + path = "/api/graphql"; + }; + # One create per run; the second finds the pull request open, so only + # the first enables auto-merge. + lines = map builtins.toJSON; + requests = entries: lib.concatMapStrings (e: e + "\n") (lines entries); + openPulls = repo: query: { + method = "GET"; + auth = "Bearer secret-token"; + body = null; + path = "/api/v3/repos/acme/${repo}/pulls?state=open${query}"; + }; + perInput = + name: + lib.recursiveUpdate (create "github" "perinput") { + body = { + head = "flake-update-${name}"; + title = "Update ${name}"; + }; + }; + mergeClean = { + method = "PUT"; + auth = "Bearer secret-token"; + body.merge_method = "rebase"; + path = "/api/v3/repos/acme/clean/pulls/1/merge"; + }; + giteaMerge = { + method = "POST"; + auth = "token secret-token"; + body = { + Do = "squash"; + delete_branch_after_merge = true; + merge_when_checks_succeed = true; + }; + path = "/api/v1/repos/acme/giteamerge/pulls/1/merge"; + }; + cases = [ + { + forge = "github"; + effect = effectFor "github" { baseMergeBranch = "develop"; }; + expected = requests [ + (create "github" "widget") + (create "github" "widget") + ]; + } + { + forge = "gitea"; + effect = effectFor "gitea" { baseMergeBranch = "develop"; }; + expected = requests [ + (create "gitea" "widget") + (create "gitea" "widget") + ]; + } + { + # GitLab leaves the base to the repository's default branch. + forge = "gitlab"; + effect = effectFor "gitlab" { }; + resets = false; + expected = requests [ + (create "gitlab" "widget") + (create "gitlab" "widget") + ]; + } + { + forge = "github"; + repo = "automerge"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + autoMergeMethod = "squash"; + }; + # The second run enables it again on the open pull request. + expected = requests [ + (create "github" "automerge") + (autoMerge "automerge" "squash") + (create "github" "automerge") + (openPulls "automerge" "&head=acme%3Aflake-update") + (autoMerge "automerge" "squash") + ]; + } + { + # Already mergeable: merged at once, as upstream does. + forge = "github"; + repo = "clean"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + autoMergeMethod = "rebase"; + }; + expected = requests [ + (create "github" "clean") + (autoMerge "clean" "rebase") + mergeClean + (create "github" "clean") + (openPulls "clean" "&head=acme%3Aflake-update") + (autoMerge "clean" "rebase") + mergeClean + ]; + } + { + forge = "gitea"; + repo = "giteamerge"; + effect = effectFor "gitea" { + baseMergeBranch = "develop"; + autoMergeMethod = "squash"; + }; + expected = requests [ + (create "gitea" "giteamerge") + giteaMerge + (create "gitea" "giteamerge") + ( + openPulls "giteamerge" "" + // { + auth = "token secret-token"; + path = "/api/v1/repos/acme/giteamerge/pulls?state=open"; + } + ) + giteaMerge + ]; + } + { + # Only the inputs not excluded are updated. + forge = "github"; + repo = "exclude"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + excludeInputs = [ "nixpkgs" ]; + }; + check = '' + grep -q ' utils --commit-lock-file' "$TMPDIR/nix-args" + ! grep -q nixpkgs "$TMPDIR/nix-args" + ''; + expected = requests [ + (create "github" "exclude") + (create "github" "exclude") + ]; + } + { + # A branch and pull request per input of the lock file. + forge = "github"; + repo = "perinput"; + effect = effectFor "github" { + perInput = true; + pullRequestTitle = null; + }; + stable = true; + branches = [ + "flake-update-nixpkgs" + "flake-update-utils" + ]; + expected = requests [ + (perInput "nixpkgs") + (perInput "utils") + (perInput "nixpkgs") + (perInput "utils") + ]; + } + ]; +in +pkgs.runCommand "effects-lib-flake-update" + { + nativeBuildInputs = [ + pkgs.git + pkgs.jq + ]; + } + '' + export HOME=$TMPDIR/home IN_HERCULES_CI_EFFECT=true + # Who seeds the test repositories. On Linux builders git cannot guess. + export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@example.com + export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@example.com + export HERCULES_CI_SECRETS_JSON=$TMPDIR/secrets.json + echo '{ "token": { "data": { "token": "secret-token" } } }' >"$HERCULES_CI_SECRETS_JSON" + source ${hook}/nix-support/setup-hook + ${fakeForge} "$TMPDIR/port" "$TMPDIR/requests" & + # A failing case must not leave the build waiting on the forge. + trap 'kill %1' EXIT + while [[ ! -s $TMPDIR/port ]]; do sleep 0.1; done + port=$(cat "$TMPDIR/port") + + ${lib.concatMapStrings run cases} + touch $out + '' diff --git a/docs/EFFECTS.md b/docs/EFFECTS.md index 75bb8779..2386d859 100644 --- a/docs/EFFECTS.md +++ b/docs/EFFECTS.md @@ -435,6 +435,38 @@ derivation attribute; effects built without `mkEffect` can set the attribute directly. Effects that do not set it get no checkout. If the repository has no forge token to push with, the effect fails with an error. +The forge token can push only if the GitHub App installation may write the +repository's contents. Where it may not, push with a secret of your own instead: +drop the clone's `url.*.insteadOf` entries, which route every URL on the forge +through the forge token, and authenticate with the secret. + +## Updating flake.lock + +effects-lib has hercules-ci-effects' `flakeUpdate`, with the same arguments. Run +it from a schedule: + +```nix +herculesCI = { primaryRepo, ... }: { + onSchedule.flake-update = { + when.dayOfWeek = [ "Mon" ]; + outputs.effects.update = (nixbot.lib.effects { inherit pkgs; }).flakeUpdate { + inherit (primaryRepo) forgeType; # github, gitea or gitlab + autoMergeMethod = "squash"; # GitHub and Gitea + }; + }; +}; +``` + +It updates the lock on the `flake-update` branch and opens a pull request to the +default branch. `baseMergeBranch` and `baseMergeMethod` (merge, rebase, +fast-forward or reset) bring an existing branch up to date first. It pushes with +`tokenSecret`, the forge token by default. Name another secret if the forge +token may not push. + +With `perInput = true`, each input gets its own branch and pull request, like +Dependabot. Runs where an input did not move leave its branch alone, so CI does +not run again. `excludeInputs` names inputs to skip. + ## Tag pushes Pushing a tag runs the tagged commit's `onPush` effects with `primaryRepo.tag` diff --git a/herculesCI/effects-lib.nix b/herculesCI/effects-lib.nix index e9a28c86..b16f1bd0 100644 --- a/herculesCI/effects-lib.nix +++ b/herculesCI/effects-lib.nix @@ -202,4 +202,10 @@ lib.fix (effects: { buildDependenciesOnly = true; }; }; + + # hercules-ci-effects' flakeUpdate, see ./flake-update.nix. + flakeUpdate = import ./flake-update.nix { + inherit pkgs lib; + inherit (effects) mkEffect; + }; }) diff --git a/herculesCI/flake-update.nix b/herculesCI/flake-update.nix new file mode 100644 index 00000000..d2a00d34 --- /dev/null +++ b/herculesCI/flake-update.nix @@ -0,0 +1,107 @@ +# Update flake.lock on a branch and open a pull request for it: the +# counterpart of hercules-ci-effects' `flakeUpdate`, taking the same +# arguments and behaving as it does. It runs in nixbot's checkout, but +# like upstream fetches, pushes and opens the pull request with +# `tokenSecret` (the forge token by default): a forge token may only +# read, so a flake that pushes with another token names it there. Pull +# requests work on GitHub, Gitea and GitLab; auto-merge on GitHub and +# Gitea. The work is done by nixbot-flake-update.py. +{ + pkgs, + lib ? pkgs.lib, + mkEffect, +}: +let + script = pkgs.writers.writePython3Bin "nixbot-flake-update" { + # ruff format owns the layout. + flakeIgnore = [ + "E203" + "E501" + "W503" + ]; + } (builtins.readFile ./nixbot-flake-update.py); +in +passedArgs@{ + # The checkout's origin if null. + gitRemote ? null, + # The user the token authenticates as; the forge's token user if null. + user ? null, + tokenSecret ? { + type = "GitToken"; + }, + updateBranch ? "flake-update", + forgeType ? "github", + createPullRequest ? true, + autoMergeMethod ? null, + # Upstream's: "`flake.lock`: Update", naming the flakes it updates. + pullRequestTitle ? null, + pullRequestBody ? null, + # As upstream: with a branch, an existing update branch is brought up to + # date with it by `baseMergeMethod` (merge, rebase, fast-forward or + # reset) before updating; without one it is updated as it stands. A new + # update branch starts from this branch, or the default branch. Pull + # requests go to the default branch either way. + baseMergeBranch ? null, + baseMergeMethod ? "merge", + flakes ? { + "." = { inherit inputs excludeInputs commitSummary; }; + }, + inputs ? [ ], + # Not upstream: inputs of the lock file to leave alone. + excludeInputs ? [ ], + commitSummary ? "", + # Not upstream: a branch `-` and pull request per + # input (of `inputs`, or all), each started afresh from the base. + perInput ? false, + nix ? pkgs.nix, +}: +assert lib.assertOneOf "forgeType" forgeType [ + "github" + "gitea" + "gitlab" +]; +assert lib.assertOneOf "baseMergeMethod" baseMergeMethod [ + "merge" + "rebase" + "fast-forward" + "reset" +]; +assert lib.assertOneOf "autoMergeMethod" autoMergeMethod [ + null + "merge" + "rebase" + "squash" +]; +assert lib.assertMsg ( + autoMergeMethod != null -> forgeType != "gitlab" +) "flakeUpdate: autoMergeMethod needs forgeType github or gitea"; +assert passedArgs ? flakes -> inputs == [ ] && excludeInputs == [ ] && commitSummary == ""; +assert flakes != { }; +let + config = pkgs.writeText "flake-update.json" ( + builtins.toJSON { + inherit + gitRemote + user + updateBranch + forgeType + createPullRequest + autoMergeMethod + pullRequestTitle + pullRequestBody + baseMergeBranch + baseMergeMethod + flakes + perInput + ; + nix = lib.getExe nix; + } + ); +in +mkEffect { + name = "flake-update"; + checkout = true; + secretsMap.token = tokenSecret; + inputs = [ pkgs.git ]; + effectScript = "${lib.getExe script} ${config}"; +} diff --git a/herculesCI/nixbot-flake-update.py b/herculesCI/nixbot-flake-update.py new file mode 100644 index 00000000..625e4762 --- /dev/null +++ b/herculesCI/nixbot-flake-update.py @@ -0,0 +1,425 @@ +"""Update flake.lock on a branch and open a pull request for it. + +The effect behind effects-lib's `flakeUpdate`, configured by the JSON +file it is given (see flake-update.nix). It runs in nixbot's checkout, +fetches and pushes with the effect's `token` secret, and opens pull +requests on GitHub, Gitea and GitLab, as hercules-ci-effects' +flakeUpdate does. With `perInput`, each input gets its own branch and +pull request, as update-flake-inputs does. +""" + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +class Failed(Exception): + """The effect fails with this message.""" + + +ERRORS = (Failed, subprocess.CalledProcessError, urllib.error.URLError) + + +def git(*args: str, check: bool = True) -> str: + result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) + if check and result.returncode != 0: + raise Failed(f"git {' '.join(args)}: {result.stderr.strip()}") + return result.stdout.strip() + + +def has_ref(ref: str) -> bool: + return ( + subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", ref], + capture_output=True, + check=False, + ).returncode + == 0 + ) + + +def token() -> str: + with open(os.environ["HERCULES_CI_SECRETS_JSON"]) as f: + return json.load(f)["token"]["data"]["token"] + + +class Forge: + """The forge behind origin: its git remote and its pull request API.""" + + def __init__(self, config: dict) -> None: + self.kind = config["forgeType"] + remote = config["gitRemote"] or git("config", "--get", "remote.origin.url") + parts = urllib.parse.urlsplit(remote) + self.host = parts.hostname or "" + if parts.port: + self.host += f":{parts.port}" + self.url = f"{parts.scheme}://{self.host}" + self.remote = parts._replace(netloc=self.host).geturl() + self.path = parts.path.strip("/").removesuffix(".git") + self.user = config["user"] or ( + "oauth2" if self.kind == "gitlab" else "x-access-token" + ) + self.token = token() + + def setup_git(self) -> None: + """Fetch and push with the token, not the checkout's. Its + insteadOf entries would put that one in every forge URL.""" + keys = git( + "config", + "--local", + "--name-only", + "--get-regexp", + r"^url\..*\.insteadof$", + check=False, + ) + # A key with several values is listed once per value. + for key in dict.fromkeys(keys.splitlines()): + git("config", "--unset-all", key) + git("remote", "set-url", "origin", self.remote) + # A credential store rather than a URL, so git cannot echo it. + scheme = self.url.split("://", 1)[0] + credentials = Path.home() / ".git-credentials" + with credentials.open("a") as f: + print(f"{scheme}://{self.user}:{self.token}@{self.host}", file=f) + git("config", "--global", "credential.helper", "store") + if self.kind == "github": + conf = Path.home() / ".config/nix/nix.conf" + conf.parent.mkdir(parents=True, exist_ok=True) + with conf.open("a") as f: + print(f"access-tokens = {self.host}={self.token}", file=f) + + def _send(self, url: str, body: dict | None, method: str = "POST") -> Any: + scheme = "token" if self.kind == "gitea" else "Bearer" + req = urllib.request.Request( + url, + data=None if body is None else json.dumps(body).encode(), + method=method, + headers={ + "Authorization": f"{scheme} {self.token}", + "Content-Type": "application/json", + # Cloudflare rejects urllib's default user agent. + "User-Agent": "nixbot-flake-update", + }, + ) + with urllib.request.urlopen(req, timeout=60) as response: + return json.loads(response.read() or b"{}") + + def _github_api(self) -> str: + if self.url == "https://github.com": + return "https://api.github.com" + return f"{self.url}/api/v3" + + def _create_request( + self, head: str, base: str, title: str, body: str + ) -> tuple[str, dict]: + if self.kind == "gitlab": + project = urllib.parse.quote(self.path, safe="") + return ( + f"{self.url}/api/v4/projects/{project}/merge_requests", + { + "title": title, + "source_branch": head, + "target_branch": base, + "description": body, + }, + ) + api = self._github_api() if self.kind == "github" else f"{self.url}/api/v1" + return ( + f"{api}/repos/{self.path}/pulls", + {"title": title, "head": head, "base": base, "body": body}, + ) + + def open_pull_request( + self, head: str, base: str, title: str, body: str + ) -> dict | None: + """The new pull request, or None if one is open already or there + is nothing to merge.""" + url, request = self._create_request(head, base, title, body) + try: + pr = self._send(url, request) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + exists = ( + e.code == 422 and ("already exists" in detail or "No commits" in detail) + if self.kind == "github" + else e.code == 409 + ) + if not exists: + raise Failed(f"opening a pull request: {e}: {detail}") from e + print(f"No new pull request needed: {detail}") + return None + print(f"Opened {pr.get('html_url') or pr.get('web_url') or pr['number']}") + return pr + + def find_open_pull_request(self, head: str) -> dict | None: + api = self._github_api() if self.kind == "github" else f"{self.url}/api/v1" + query = "state=open" + if self.kind == "github": + owner = self.path.split("/", 1)[0] + query += f"&head={urllib.parse.quote(f'{owner}:{head}')}" + pulls = self._send(f"{api}/repos/{self.path}/pulls?{query}", None, "GET") + return next((p for p in pulls if p["head"]["ref"] == head), None) + + def auto_merge(self, pr: dict, method: str) -> None: + if self.kind == "gitea": + self._gitea_auto_merge(pr, method) + else: + self._github_auto_merge(pr, method) + + def _gitea_auto_merge(self, pr: dict, method: str) -> None: + url = f"{self.url}/api/v1/repos/{self.path}/pulls/{pr['number']}/merge" + request = { + "Do": method, + "merge_when_checks_succeed": True, + "delete_branch_after_merge": True, + } + # Gitea answers "Please try again later" while it computes the + # pull request's mergeability. + for _ in range(5): + try: + reply = self._send(url, request) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + if e.code == 409: + print(f"#{pr['number']}: {detail}") + return + raise Failed(f"enabling auto-merge: {e}: {detail}") from e + if reply.get("message") != "Please try again later": + print(f"Scheduled #{pr['number']} to merge ({method})") + return + time.sleep(2) + raise Failed(f"Gitea kept #{pr['number']} from merging") + + def _github_auto_merge(self, pr: dict, method: str) -> None: + """Enable auto-merge, or merge when GitHub says the pull request + is in clean status, as hercules-ci-effects does.""" + if self.url == "https://github.com": + graphql = "https://api.github.com/graphql" + else: + graphql = f"{self.url}/api/graphql" + query = ( + "mutation ($id: ID!, $method: PullRequestMergeMethod!) {" + " enablePullRequestAutoMerge(input: {pullRequestId: $id," + " mergeMethod: $method}) { clientMutationId } }" + ) + variables = {"id": pr["node_id"], "method": method.upper()} + reply = self._send(graphql, {"query": query, "variables": variables}) + errors = reply.get("errors") or [] + if not errors: + print(f"Enabled auto-merge ({method}) on #{pr['number']}") + elif any("clean status" in e.get("message", "") for e in errors): + print(f"#{pr['number']} is already in clean status. Merging.") + url = f"{self._github_api()}/repos/{self.path}/pulls/{pr['number']}/merge" + self._send(url, {"merge_method": method}, method="PUT") + else: + raise Failed( + f"enabling auto-merge failed: {errors}\nAuto-merge must " + "be allowed in the repository settings, and the base " + "branch needs a protection rule with required checks." + ) + + +def nix_flake_update(nix: str, path: str, inputs: list[str], summary: str) -> None: + command = [ + nix, + "--extra-experimental-features", + "nix-command flakes", + "flake", + "update", + *inputs, + "--commit-lock-file", + ] + if summary: + command += ["--commit-lockfile-summary", summary] + # Answer "n" to accepting nixConfig, as hercules-ci-effects does. + subprocess.run(command, cwd=path, input="n\n" * 100, text=True, check=True) + + +def flake_inputs(rev: str, path: str, flake: dict) -> list[str]: + """The inputs to update: those named, or the lock file's direct + inputs, less the excluded ones.""" + inputs = flake.get("inputs") or [] + excluded = flake.get("excludeInputs") or [] + if not inputs and (excluded or rev != "HEAD"): + lock = Path(path, "flake.lock").as_posix() + if not has_ref(f"{rev}:{lock}"): + print(f"Skipping {path}: it has no flake.lock", file=sys.stderr) + return [] + locks = json.loads(git("show", f"{rev}:{lock}")) + inputs = sorted(locks["nodes"][locks["root"]].get("inputs", {})) + return [i for i in inputs if i not in excluded] + + +def identity(rev: str) -> str: + """A commit, less its dates and people.""" + return git("log", "-1", "--format=%T%n%P%n%B", rev) + + +def merge_base(update_branch: str, base: str, method: str | None) -> None: + """Check out the update branch: brought up to date with the base by + `method`, or as it stands without one. A new branch, or one that is + reset, starts from the base.""" + update = f"refs/remotes/origin/{update_branch}" + if not has_ref(update) or method == "reset": + git("checkout", "--quiet", "-B", update_branch, base) + return + git("checkout", "--quiet", "-B", update_branch, update) + if method is None: + return + command = { + "merge": ["merge", "--no-edit", base], + "rebase": ["rebase", base], + "fast-forward": ["merge", "--ff-only", base], + }[method] + if subprocess.run(["git", *command], check=False).returncode == 0: + return + if method == "fast-forward": + raise Failed( + f"{update_branch} has commits the base lacks, so it cannot " + f"fast-forward. Delete it (git push origin :{update_branch})" + " or merge it, or use another baseMergeMethod." + ) + conflicts = git("diff", "--name-only", "--diff-filter=U") + raise Failed( + f"Resolve the conflicts by hand and push to {update_branch}:\n{conflicts}" + ) + + +def push(update_branch: str, before: str, force: str | None) -> bool: + """Push the update branch if the update committed anything.""" + if git("diff", "HEAD", "--stat"): + raise Failed("the update left uncommitted changes") + if git("rev-parse", "HEAD") == before: + print(f"No updates to push to {update_branch}.") + return False + # A rebuilt branch differs from the pushed one only in its dates + # unless an input moved: pushing it would rerun CI for nothing. + remote = f"refs/remotes/origin/{update_branch}" + if has_ref(remote) and identity("HEAD") == identity(remote): + print(f"{update_branch} is up to date.") + return True + flags = [f"--{force}"] if force else [] + git("push", "origin", f"HEAD:refs/heads/{update_branch}", *flags) + return True + + +def open_pull_request( + forge: Forge, config: dict, update_branch: str, base: str, title: str +) -> None: + update = f"refs/remotes/origin/{update_branch}" + # Too many pull requests beats too few: make sure one is open. + if not config["createPullRequest"] or not has_ref(update): + return + body = config["pullRequestBody"] + if body is None: + body = git("log", "-1", "--format=%b", update) + pr = forge.open_pull_request(update_branch, base, title, body) + method = config["autoMergeMethod"] + if method is None: + return + # A push by someone else turns GitHub's auto-merge off again. + pr = pr or forge.find_open_pull_request(update_branch) + if pr is not None: + forge.auto_merge(pr, method) + + +def title_for(paths: list[str]) -> str: + """As hercules-ci-effects names it.""" + names = ["`flake.lock`" if p == "." else f"`{p}/flake.lock`" for p in paths] + return "`flake.lock`" if len(names) > 3 else ", ".join(names) + + +def update_together(forge: Forge, config: dict, default_branch: str, nix: str) -> None: + """One branch and pull request for every flake, as upstream.""" + branch = config["updateBranch"] + base_branch = config["baseMergeBranch"] + # Upstream's baseMerge.enable: a method applies only with a branch. + method = config["baseMergeMethod"] if base_branch else None + merge_base(branch, f"refs/remotes/origin/{base_branch or default_branch}", method) + before = git("rev-parse", "HEAD") + flakes = config["flakes"] + for path, flake in flakes.items(): + inputs = flake_inputs("HEAD", path, flake) + if not inputs and (flake.get("inputs") or flake.get("excludeInputs")): + continue + where = "" if list(flakes) == ["."] else f" in '{path}'" + print(f"Running nix flake update{where}...", file=sys.stderr) + nix_flake_update(nix, path, inputs, flake.get("commitSummary", "")) + # Rebase rewrites the branch, and reset starts it over. + force = {"rebase": "force-with-lease", "reset": "force"}.get(method or "") + push(branch, before, force) + title = config["pullRequestTitle"] or f"{title_for(list(flakes))}: Update" + open_pull_request(forge, config, branch, default_branch, title) + + +def update_each(forge: Forge, config: dict, default_branch: str, nix: str) -> None: + """A branch and pull request per input, each started afresh from the + base. One failing input does not stop the others.""" + base = f"refs/remotes/origin/{config['baseMergeBranch'] or default_branch}" + failed = [] + for path, flake in config["flakes"].items(): + inputs = flake_inputs(base, path, flake) + where = "" if path == "." else f" in {path}" + for name in inputs: + suffix = "" if path == "." else f"-{path}" + branch = f"{config['updateBranch']}{suffix}-{name}".replace("/", "-") + summary = f"Update {name}{where}" + try: + print(f"{summary}...", file=sys.stderr) + git("checkout", "--quiet", "-B", branch, base) + before = git("rev-parse", "HEAD") + nix_flake_update(nix, path, [name], summary) + if not push(branch, before, "force") and has_ref( + f"refs/remotes/origin/{branch}" + ): + # The base has the update already. + git("push", "origin", f":refs/heads/{branch}") + continue + open_pull_request( + forge, + config, + branch, + default_branch, + config["pullRequestTitle"] or summary, + ) + except ERRORS as e: + print(f"{summary} failed: {e}", file=sys.stderr) + failed.append(summary) + if failed: + raise Failed("failed: " + ", ".join(failed)) + + +def run(config: dict) -> None: + for who in ("AUTHOR", "COMMITTER"): + os.environ.setdefault(f"GIT_{who}_NAME", "nixbot") + os.environ.setdefault(f"GIT_{who}_EMAIL", "nixbot@localhost") + forge = Forge(config) + forge.setup_git() + head = git("ls-remote", "--symref", "origin", "HEAD") + default_branch = head.split("\t", 1)[0].removeprefix("ref: refs/heads/") + # The checkout comes from nixbot's mirror. The forge has the + # branches as they are now. + git("fetch", "--quiet", "origin", "+refs/heads/*:refs/remotes/origin/*") + if config["perInput"]: + update_each(forge, config, default_branch, config["nix"]) + else: + update_together(forge, config, default_branch, config["nix"]) + + +def main() -> None: + with open(sys.argv[1]) as f: + config = json.load(f) + try: + run(config) + except ERRORS as e: + sys.exit(f"flakeUpdate: {e}") + + +main()