From 1315830d68b7e811a9b8e7561481782c8fc962be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20Bjarki=20Bogason?= Date: Sun, 4 Oct 2026 10:19:23 +0000 Subject: [PATCH] effects-lib: add hercules-ci-effects' flakeUpdate Flakes that keep their lock current from a scheduled effect use hci-effects.flakeUpdate, which keeps hercules-ci-effects as an input. Add one with the same arguments and behaviour, opening pull requests on GitHub, Gitea and GitLab, and pushing with tokenSecret since the checkout's forge token may only read. With perInput, each input gets its own branch and pull request, as with update-flake-inputs. Like it, the effect skips pushes that would only change commit dates, so CI does not rerun on every schedule, supports Gitea auto-merge and excluding inputs, and re-enables auto-merge on pull requests that are already open. --- checks/default.nix | 1 + checks/docs-examples.nix | 1 + checks/effects-lib-flake-update.nix | 432 ++++++++++++++++++++++++++++ docs/EFFECTS.md | 32 +++ herculesCI/effects-lib.nix | 6 + herculesCI/flake-update.nix | 107 +++++++ herculesCI/nixbot-flake-update.py | 425 +++++++++++++++++++++++++++ 7 files changed, 1004 insertions(+) create mode 100644 checks/effects-lib-flake-update.nix create mode 100644 herculesCI/flake-update.nix create mode 100644 herculesCI/nixbot-flake-update.py diff --git a/checks/default.nix b/checks/default.nix index b1d81180..f4072624 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -17,6 +17,7 @@ in nixbot-effects-tests = self.packages.${pkgs.stdenv.hostPlatform.system}.nixbot-effects.tests.pytest; effects-lib-ssh = import ./effects-lib-ssh.nix checkArgs; effects-lib-run-nixos = import ./effects-lib-run-nixos.nix checkArgs; + effects-lib-flake-update = import ./effects-lib-flake-update.nix checkArgs; sqlc-generated = import ./sqlc.nix checkArgs; effects-lib = import ./effects-lib.nix checkArgs; docs-examples = import ./docs-examples.nix checkArgs; diff --git a/checks/docs-examples.nix b/checks/docs-examples.nix index c1ccb179..77ea7bfb 100644 --- a/checks/docs-examples.nix +++ b/checks/docs-examples.nix @@ -74,6 +74,7 @@ let primaryRepo = tag: { rev = "0123456789abcdef"; + forgeType = "github"; branch = if tag == null then "main" else null; inherit tag; }; diff --git a/checks/effects-lib-flake-update.nix b/checks/effects-lib-flake-update.nix new file mode 100644 index 00000000..f82682ee --- /dev/null +++ b/checks/effects-lib-flake-update.nix @@ -0,0 +1,432 @@ +# effects-lib's `flakeUpdate`: its setup and effect scripts run in a +# checkout set up as nixbot sets one up, with the checkout's token in +# origin and in an insteadOf for the forge. The forge is local bare +# repositories behind an http URL (a global insteadOf that only applies +# once the effect has dropped the checkout's) and a fake API at it. The +# stand-in nix rewrites flake.lock and commits it like +# `nix flake update --commit-lock-file`. +{ pkgs, ... }: +let + inherit (pkgs) lib; + effects = import ../herculesCI/effects-lib.nix { inherit pkgs; }; + fakeNix = pkgs.writeShellScriptBin "nix" '' + printf '%s\n' "$*" >>"$TMPDIR/nix-args" + # Stable: the same update every run, as when no input moved. + if [[ -n ''${FAKE_NIX_STABLE:-} ]]; then lock="$*"; else lock=$RANDOM$RANDOM; fi + echo "{ \"run\": \"$lock\" }" >flake.lock + git commit -q -m "flake.lock: Update" -m "Updated input 'nixpkgs'" flake.lock + ''; + fakeForge = pkgs.writers.writePython3 "fake-forge" { } '' + import http.server + import json + import re + import sys + + # The forges' replies to a second pull request for the same branch. + EXISTS = { + "/api/v3/repos/acme/{}/pulls": (422, { + "message": "Validation Failed", + "errors": [{"message": "A pull request already exists for " + "acme:flake-update."}], + }), + "/api/v1/repos/acme/{}/pulls": (409, { + "message": "pull request already exists for these targets", + }), + "/api/v4/projects/acme%2F{}/merge_requests": (409, { + "message": ["Another open merge request already exists for " + "this source branch: !1"], + }), + } + seen = set() + + + class Forge(http.server.BaseHTTPRequestHandler): + def reply(self, status, body): + self.send_response(status) + self.end_headers() + self.wfile.write(json.dumps(body).encode()) + + def record(self): + length = int(self.headers["Content-Length"] or 0) + body = json.loads(self.rfile.read(length)) if length else None + with open(sys.argv[2], "a") as log: + entry = {"method": self.command, "path": self.path, + "body": body, "auth": self.headers["Authorization"]} + print(json.dumps(entry, sort_keys=True, separators=(",", ":")), + file=log) + return body + + def do_POST(self): + body = self.record() + if self.path == "/api/graphql": + # The "clean" repository's pull request is mergeable now. + if body["variables"]["id"] == "PR_clean": + return self.reply(200, {"errors": [{ + "message": "Pull request is in clean status"}]}) + return self.reply(200, {"data": {}}) + if self.path.endswith("/merge"): + return self.reply(200, {}) + repo = re.search(r"acme(?:/|%2F)([^/]+)", self.path)[1] + template = self.path.replace(repo, "{}", 1) + pr = (self.path, body.get("head") or body["source_branch"]) + if pr in seen: + return self.reply(*EXISTS[template]) + seen.add(pr) + self.reply(201, {"number": 1, "node_id": f"PR_{repo}"}) + + def do_GET(self): + self.record() + path = self.path.split("?", 1)[0] + repo = re.search(r"acme/([^/]+)", path)[1] + self.reply(200, [ + {"number": 1, "node_id": f"PR_{repo}", "head": {"ref": head}} + for p, head in seen if p == path + ]) + + def do_PUT(self): + self.record() + self.reply(200, {"merged": True}) + + def log_message(self, *args): + pass + + + server = http.server.HTTPServer(("127.0.0.1", 0), Forge) + with open(sys.argv[1], "w") as f: + f.write(str(server.server_port)) + server.serve_forever() + ''; + # Two root inputs, which perInput updates one by one. + seedLock = builtins.toJSON { + nodes.root.inputs = { + nixpkgs = "nixpkgs"; + utils = "utils"; + }; + root = "root"; + version = 7; + }; + effectFor = + forgeType: args: + effects.flakeUpdate ( + { + inherit forgeType; + nix = fakeNix; + baseMergeMethod = "reset"; + pullRequestTitle = "chore: update flake.lock"; + } + // args + ); + hook = + lib.findFirst (p: lib.getName p == "hercules-ci-effect-sh") (throw "no setup hook") + (effects.mkEffect { }).nativeBuildInputs; + run = + { + forge, + repo ? "widget", + effect, + # Without a baseMergeBranch the update branch is updated as it stands. + resets ? true, + branches ? [ "flake-update" ], + # Each run updates the same: the second must not push. + stable ? false, + check ? "", + expected, + }: + '' + remote=$TMPDIR/forge/acme/${repo}.git + rm -rf "$TMPDIR/forge" "$TMPDIR/checkout" "$TMPDIR/requests" "$HOME" + mkdir -p "$HOME" + git init -q --bare -b develop "$remote" + git clone -q "$remote" "$TMPDIR/seed" 2>/dev/null + (cd "$TMPDIR/seed" && echo ${lib.escapeShellArg seedLock} >flake.lock && git add flake.lock \ + && git commit -qm init && git push -q origin HEAD:develop) + rm -rf "$TMPDIR/seed" + develop=$(git -C "$remote" rev-parse develop) + + # The forge's git side. git takes the longest matching insteadOf, so + # while the checkout's (one character longer) is there, URLs go to + # the fake API with its token and fail. + forge="http://127.0.0.1:$port" + git config --global url."$TMPDIR/forge".insteadOf "$forge" + # nixbot's checkout: its token in origin and in every forge URL. + git clone -q "$remote" "$TMPDIR/checkout" + git -C "$TMPDIR/checkout" remote set-url origin \ + "http://x-access-token:checkout-token@127.0.0.1:$port/acme/${repo}.git" + # Two values under one key, as nixbot sets for https and ssh. + for prefix in "$forge/" "git@127.0.0.1:"; do + git -C "$TMPDIR/checkout" config --add \ + url."http://x-access-token:checkout-token@127.0.0.1:$port/".insteadOf "$prefix" + done + + for attempt in 1 2; do + heads=$(git -C "$remote" for-each-ref --format='%(objectname)' refs/heads) + ( + ${lib.optionalString stable "export FAKE_NIX_STABLE=1"} + cd "$TMPDIR/checkout" + export PATH=${lib.makeBinPath effect.nativeBuildInputs}:$PATH + export NIXBOT_EFFECT_CHECKOUT=$TMPDIR/checkout + set -euo pipefail + eval ${lib.escapeShellArg effect.userSetupScript} + eval ${lib.escapeShellArg effect.effectScript} + ) + done + + # Fetched and pushed with the effect's token, not the checkout's. + grep -qx "http://${ + if forge == "gitlab" then "oauth2" else "x-access-token" + }:secret-token@127.0.0.1:$port" ~/.git-credentials + ${lib.optionalString (forge == "github") '' + grep -qx "access-tokens = 127.0.0.1:$port=secret-token" ~/.config/nix/nix.conf + ''} + ${lib.optionalString stable '' + [[ $(git -C "$remote" for-each-ref --format='%(objectname)' refs/heads) == "$heads" ]] + ''} + ${check} + # One update per branch and run. + [[ $(grep -c . "$TMPDIR/nix-args") == ${toString (2 * builtins.length branches)} ]] + rm "$TMPDIR/nix-args" + for branch in ${lib.escapeShellArgs branches}; do + ${ + if resets then + '' + # Reset: each run starts again from develop and force-pushes. + [[ $(git -C "$remote" rev-parse "$branch^") == "$develop" ]] + '' + else + '' + # No base merge: the second run updates the branch as it stands. + [[ $(git -C "$remote" rev-parse "$branch~2") == "$develop" ]] + '' + } + done + diff "$TMPDIR/requests" ${pkgs.writeText "${forge}-${repo}-requests" expected} + ''; + create = + forge: repo: + { + github = { + method = "POST"; + auth = "Bearer secret-token"; + body = { + base = "develop"; + body = "Updated input 'nixpkgs'"; + head = "flake-update"; + title = "chore: update flake.lock"; + }; + path = "/api/v3/repos/acme/${repo}/pulls"; + }; + gitea = { + method = "POST"; + auth = "token secret-token"; + body = { + base = "develop"; + body = "Updated input 'nixpkgs'"; + head = "flake-update"; + title = "chore: update flake.lock"; + }; + path = "/api/v1/repos/acme/${repo}/pulls"; + }; + gitlab = { + method = "POST"; + auth = "Bearer secret-token"; + body = { + description = "Updated input 'nixpkgs'"; + source_branch = "flake-update"; + target_branch = "develop"; + title = "chore: update flake.lock"; + }; + path = "/api/v4/projects/acme%2F${repo}/merge_requests"; + }; + } + .${forge}; + autoMerge = repo: method: { + method = "POST"; + auth = "Bearer secret-token"; + body = { + query = "mutation ($id: ID!, $method: PullRequestMergeMethod!) { enablePullRequestAutoMerge(input: {pullRequestId: $id, mergeMethod: $method}) { clientMutationId } }"; + variables = { + id = "PR_${repo}"; + method = lib.toUpper method; + }; + }; + path = "/api/graphql"; + }; + # One create per run; the second finds the pull request open, so only + # the first enables auto-merge. + lines = map builtins.toJSON; + requests = entries: lib.concatMapStrings (e: e + "\n") (lines entries); + openPulls = repo: query: { + method = "GET"; + auth = "Bearer secret-token"; + body = null; + path = "/api/v3/repos/acme/${repo}/pulls?state=open${query}"; + }; + perInput = + name: + lib.recursiveUpdate (create "github" "perinput") { + body = { + head = "flake-update-${name}"; + title = "Update ${name}"; + }; + }; + mergeClean = { + method = "PUT"; + auth = "Bearer secret-token"; + body.merge_method = "rebase"; + path = "/api/v3/repos/acme/clean/pulls/1/merge"; + }; + giteaMerge = { + method = "POST"; + auth = "token secret-token"; + body = { + Do = "squash"; + delete_branch_after_merge = true; + merge_when_checks_succeed = true; + }; + path = "/api/v1/repos/acme/giteamerge/pulls/1/merge"; + }; + cases = [ + { + forge = "github"; + effect = effectFor "github" { baseMergeBranch = "develop"; }; + expected = requests [ + (create "github" "widget") + (create "github" "widget") + ]; + } + { + forge = "gitea"; + effect = effectFor "gitea" { baseMergeBranch = "develop"; }; + expected = requests [ + (create "gitea" "widget") + (create "gitea" "widget") + ]; + } + { + # GitLab leaves the base to the repository's default branch. + forge = "gitlab"; + effect = effectFor "gitlab" { }; + resets = false; + expected = requests [ + (create "gitlab" "widget") + (create "gitlab" "widget") + ]; + } + { + forge = "github"; + repo = "automerge"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + autoMergeMethod = "squash"; + }; + # The second run enables it again on the open pull request. + expected = requests [ + (create "github" "automerge") + (autoMerge "automerge" "squash") + (create "github" "automerge") + (openPulls "automerge" "&head=acme%3Aflake-update") + (autoMerge "automerge" "squash") + ]; + } + { + # Already mergeable: merged at once, as upstream does. + forge = "github"; + repo = "clean"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + autoMergeMethod = "rebase"; + }; + expected = requests [ + (create "github" "clean") + (autoMerge "clean" "rebase") + mergeClean + (create "github" "clean") + (openPulls "clean" "&head=acme%3Aflake-update") + (autoMerge "clean" "rebase") + mergeClean + ]; + } + { + forge = "gitea"; + repo = "giteamerge"; + effect = effectFor "gitea" { + baseMergeBranch = "develop"; + autoMergeMethod = "squash"; + }; + expected = requests [ + (create "gitea" "giteamerge") + giteaMerge + (create "gitea" "giteamerge") + ( + openPulls "giteamerge" "" + // { + auth = "token secret-token"; + path = "/api/v1/repos/acme/giteamerge/pulls?state=open"; + } + ) + giteaMerge + ]; + } + { + # Only the inputs not excluded are updated. + forge = "github"; + repo = "exclude"; + effect = effectFor "github" { + baseMergeBranch = "develop"; + excludeInputs = [ "nixpkgs" ]; + }; + check = '' + grep -q ' utils --commit-lock-file' "$TMPDIR/nix-args" + ! grep -q nixpkgs "$TMPDIR/nix-args" + ''; + expected = requests [ + (create "github" "exclude") + (create "github" "exclude") + ]; + } + { + # A branch and pull request per input of the lock file. + forge = "github"; + repo = "perinput"; + effect = effectFor "github" { + perInput = true; + pullRequestTitle = null; + }; + stable = true; + branches = [ + "flake-update-nixpkgs" + "flake-update-utils" + ]; + expected = requests [ + (perInput "nixpkgs") + (perInput "utils") + (perInput "nixpkgs") + (perInput "utils") + ]; + } + ]; +in +pkgs.runCommand "effects-lib-flake-update" + { + nativeBuildInputs = [ + pkgs.git + pkgs.jq + ]; + } + '' + export HOME=$TMPDIR/home IN_HERCULES_CI_EFFECT=true + # Who seeds the test repositories. On Linux builders git cannot guess. + export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@example.com + export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@example.com + export HERCULES_CI_SECRETS_JSON=$TMPDIR/secrets.json + echo '{ "token": { "data": { "token": "secret-token" } } }' >"$HERCULES_CI_SECRETS_JSON" + source ${hook}/nix-support/setup-hook + ${fakeForge} "$TMPDIR/port" "$TMPDIR/requests" & + # A failing case must not leave the build waiting on the forge. + trap 'kill %1' EXIT + while [[ ! -s $TMPDIR/port ]]; do sleep 0.1; done + port=$(cat "$TMPDIR/port") + + ${lib.concatMapStrings run cases} + touch $out + '' diff --git a/docs/EFFECTS.md b/docs/EFFECTS.md index 75bb8779..2386d859 100644 --- a/docs/EFFECTS.md +++ b/docs/EFFECTS.md @@ -435,6 +435,38 @@ derivation attribute; effects built without `mkEffect` can set the attribute directly. Effects that do not set it get no checkout. If the repository has no forge token to push with, the effect fails with an error. +The forge token can push only if the GitHub App installation may write the +repository's contents. Where it may not, push with a secret of your own instead: +drop the clone's `url.*.insteadOf` entries, which route every URL on the forge +through the forge token, and authenticate with the secret. + +## Updating flake.lock + +effects-lib has hercules-ci-effects' `flakeUpdate`, with the same arguments. Run +it from a schedule: + +```nix +herculesCI = { primaryRepo, ... }: { + onSchedule.flake-update = { + when.dayOfWeek = [ "Mon" ]; + outputs.effects.update = (nixbot.lib.effects { inherit pkgs; }).flakeUpdate { + inherit (primaryRepo) forgeType; # github, gitea or gitlab + autoMergeMethod = "squash"; # GitHub and Gitea + }; + }; +}; +``` + +It updates the lock on the `flake-update` branch and opens a pull request to the +default branch. `baseMergeBranch` and `baseMergeMethod` (merge, rebase, +fast-forward or reset) bring an existing branch up to date first. It pushes with +`tokenSecret`, the forge token by default. Name another secret if the forge +token may not push. + +With `perInput = true`, each input gets its own branch and pull request, like +Dependabot. Runs where an input did not move leave its branch alone, so CI does +not run again. `excludeInputs` names inputs to skip. + ## Tag pushes Pushing a tag runs the tagged commit's `onPush` effects with `primaryRepo.tag` diff --git a/herculesCI/effects-lib.nix b/herculesCI/effects-lib.nix index e9a28c86..b16f1bd0 100644 --- a/herculesCI/effects-lib.nix +++ b/herculesCI/effects-lib.nix @@ -202,4 +202,10 @@ lib.fix (effects: { buildDependenciesOnly = true; }; }; + + # hercules-ci-effects' flakeUpdate, see ./flake-update.nix. + flakeUpdate = import ./flake-update.nix { + inherit pkgs lib; + inherit (effects) mkEffect; + }; }) diff --git a/herculesCI/flake-update.nix b/herculesCI/flake-update.nix new file mode 100644 index 00000000..d2a00d34 --- /dev/null +++ b/herculesCI/flake-update.nix @@ -0,0 +1,107 @@ +# Update flake.lock on a branch and open a pull request for it: the +# counterpart of hercules-ci-effects' `flakeUpdate`, taking the same +# arguments and behaving as it does. It runs in nixbot's checkout, but +# like upstream fetches, pushes and opens the pull request with +# `tokenSecret` (the forge token by default): a forge token may only +# read, so a flake that pushes with another token names it there. Pull +# requests work on GitHub, Gitea and GitLab; auto-merge on GitHub and +# Gitea. The work is done by nixbot-flake-update.py. +{ + pkgs, + lib ? pkgs.lib, + mkEffect, +}: +let + script = pkgs.writers.writePython3Bin "nixbot-flake-update" { + # ruff format owns the layout. + flakeIgnore = [ + "E203" + "E501" + "W503" + ]; + } (builtins.readFile ./nixbot-flake-update.py); +in +passedArgs@{ + # The checkout's origin if null. + gitRemote ? null, + # The user the token authenticates as; the forge's token user if null. + user ? null, + tokenSecret ? { + type = "GitToken"; + }, + updateBranch ? "flake-update", + forgeType ? "github", + createPullRequest ? true, + autoMergeMethod ? null, + # Upstream's: "`flake.lock`: Update", naming the flakes it updates. + pullRequestTitle ? null, + pullRequestBody ? null, + # As upstream: with a branch, an existing update branch is brought up to + # date with it by `baseMergeMethod` (merge, rebase, fast-forward or + # reset) before updating; without one it is updated as it stands. A new + # update branch starts from this branch, or the default branch. Pull + # requests go to the default branch either way. + baseMergeBranch ? null, + baseMergeMethod ? "merge", + flakes ? { + "." = { inherit inputs excludeInputs commitSummary; }; + }, + inputs ? [ ], + # Not upstream: inputs of the lock file to leave alone. + excludeInputs ? [ ], + commitSummary ? "", + # Not upstream: a branch `-` and pull request per + # input (of `inputs`, or all), each started afresh from the base. + perInput ? false, + nix ? pkgs.nix, +}: +assert lib.assertOneOf "forgeType" forgeType [ + "github" + "gitea" + "gitlab" +]; +assert lib.assertOneOf "baseMergeMethod" baseMergeMethod [ + "merge" + "rebase" + "fast-forward" + "reset" +]; +assert lib.assertOneOf "autoMergeMethod" autoMergeMethod [ + null + "merge" + "rebase" + "squash" +]; +assert lib.assertMsg ( + autoMergeMethod != null -> forgeType != "gitlab" +) "flakeUpdate: autoMergeMethod needs forgeType github or gitea"; +assert passedArgs ? flakes -> inputs == [ ] && excludeInputs == [ ] && commitSummary == ""; +assert flakes != { }; +let + config = pkgs.writeText "flake-update.json" ( + builtins.toJSON { + inherit + gitRemote + user + updateBranch + forgeType + createPullRequest + autoMergeMethod + pullRequestTitle + pullRequestBody + baseMergeBranch + baseMergeMethod + flakes + perInput + ; + nix = lib.getExe nix; + } + ); +in +mkEffect { + name = "flake-update"; + checkout = true; + secretsMap.token = tokenSecret; + inputs = [ pkgs.git ]; + effectScript = "${lib.getExe script} ${config}"; +} diff --git a/herculesCI/nixbot-flake-update.py b/herculesCI/nixbot-flake-update.py new file mode 100644 index 00000000..625e4762 --- /dev/null +++ b/herculesCI/nixbot-flake-update.py @@ -0,0 +1,425 @@ +"""Update flake.lock on a branch and open a pull request for it. + +The effect behind effects-lib's `flakeUpdate`, configured by the JSON +file it is given (see flake-update.nix). It runs in nixbot's checkout, +fetches and pushes with the effect's `token` secret, and opens pull +requests on GitHub, Gitea and GitLab, as hercules-ci-effects' +flakeUpdate does. With `perInput`, each input gets its own branch and +pull request, as update-flake-inputs does. +""" + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +class Failed(Exception): + """The effect fails with this message.""" + + +ERRORS = (Failed, subprocess.CalledProcessError, urllib.error.URLError) + + +def git(*args: str, check: bool = True) -> str: + result = subprocess.run(["git", *args], capture_output=True, text=True, check=False) + if check and result.returncode != 0: + raise Failed(f"git {' '.join(args)}: {result.stderr.strip()}") + return result.stdout.strip() + + +def has_ref(ref: str) -> bool: + return ( + subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", ref], + capture_output=True, + check=False, + ).returncode + == 0 + ) + + +def token() -> str: + with open(os.environ["HERCULES_CI_SECRETS_JSON"]) as f: + return json.load(f)["token"]["data"]["token"] + + +class Forge: + """The forge behind origin: its git remote and its pull request API.""" + + def __init__(self, config: dict) -> None: + self.kind = config["forgeType"] + remote = config["gitRemote"] or git("config", "--get", "remote.origin.url") + parts = urllib.parse.urlsplit(remote) + self.host = parts.hostname or "" + if parts.port: + self.host += f":{parts.port}" + self.url = f"{parts.scheme}://{self.host}" + self.remote = parts._replace(netloc=self.host).geturl() + self.path = parts.path.strip("/").removesuffix(".git") + self.user = config["user"] or ( + "oauth2" if self.kind == "gitlab" else "x-access-token" + ) + self.token = token() + + def setup_git(self) -> None: + """Fetch and push with the token, not the checkout's. Its + insteadOf entries would put that one in every forge URL.""" + keys = git( + "config", + "--local", + "--name-only", + "--get-regexp", + r"^url\..*\.insteadof$", + check=False, + ) + # A key with several values is listed once per value. + for key in dict.fromkeys(keys.splitlines()): + git("config", "--unset-all", key) + git("remote", "set-url", "origin", self.remote) + # A credential store rather than a URL, so git cannot echo it. + scheme = self.url.split("://", 1)[0] + credentials = Path.home() / ".git-credentials" + with credentials.open("a") as f: + print(f"{scheme}://{self.user}:{self.token}@{self.host}", file=f) + git("config", "--global", "credential.helper", "store") + if self.kind == "github": + conf = Path.home() / ".config/nix/nix.conf" + conf.parent.mkdir(parents=True, exist_ok=True) + with conf.open("a") as f: + print(f"access-tokens = {self.host}={self.token}", file=f) + + def _send(self, url: str, body: dict | None, method: str = "POST") -> Any: + scheme = "token" if self.kind == "gitea" else "Bearer" + req = urllib.request.Request( + url, + data=None if body is None else json.dumps(body).encode(), + method=method, + headers={ + "Authorization": f"{scheme} {self.token}", + "Content-Type": "application/json", + # Cloudflare rejects urllib's default user agent. + "User-Agent": "nixbot-flake-update", + }, + ) + with urllib.request.urlopen(req, timeout=60) as response: + return json.loads(response.read() or b"{}") + + def _github_api(self) -> str: + if self.url == "https://github.com": + return "https://api.github.com" + return f"{self.url}/api/v3" + + def _create_request( + self, head: str, base: str, title: str, body: str + ) -> tuple[str, dict]: + if self.kind == "gitlab": + project = urllib.parse.quote(self.path, safe="") + return ( + f"{self.url}/api/v4/projects/{project}/merge_requests", + { + "title": title, + "source_branch": head, + "target_branch": base, + "description": body, + }, + ) + api = self._github_api() if self.kind == "github" else f"{self.url}/api/v1" + return ( + f"{api}/repos/{self.path}/pulls", + {"title": title, "head": head, "base": base, "body": body}, + ) + + def open_pull_request( + self, head: str, base: str, title: str, body: str + ) -> dict | None: + """The new pull request, or None if one is open already or there + is nothing to merge.""" + url, request = self._create_request(head, base, title, body) + try: + pr = self._send(url, request) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + exists = ( + e.code == 422 and ("already exists" in detail or "No commits" in detail) + if self.kind == "github" + else e.code == 409 + ) + if not exists: + raise Failed(f"opening a pull request: {e}: {detail}") from e + print(f"No new pull request needed: {detail}") + return None + print(f"Opened {pr.get('html_url') or pr.get('web_url') or pr['number']}") + return pr + + def find_open_pull_request(self, head: str) -> dict | None: + api = self._github_api() if self.kind == "github" else f"{self.url}/api/v1" + query = "state=open" + if self.kind == "github": + owner = self.path.split("/", 1)[0] + query += f"&head={urllib.parse.quote(f'{owner}:{head}')}" + pulls = self._send(f"{api}/repos/{self.path}/pulls?{query}", None, "GET") + return next((p for p in pulls if p["head"]["ref"] == head), None) + + def auto_merge(self, pr: dict, method: str) -> None: + if self.kind == "gitea": + self._gitea_auto_merge(pr, method) + else: + self._github_auto_merge(pr, method) + + def _gitea_auto_merge(self, pr: dict, method: str) -> None: + url = f"{self.url}/api/v1/repos/{self.path}/pulls/{pr['number']}/merge" + request = { + "Do": method, + "merge_when_checks_succeed": True, + "delete_branch_after_merge": True, + } + # Gitea answers "Please try again later" while it computes the + # pull request's mergeability. + for _ in range(5): + try: + reply = self._send(url, request) + except urllib.error.HTTPError as e: + detail = e.read().decode(errors="replace") + if e.code == 409: + print(f"#{pr['number']}: {detail}") + return + raise Failed(f"enabling auto-merge: {e}: {detail}") from e + if reply.get("message") != "Please try again later": + print(f"Scheduled #{pr['number']} to merge ({method})") + return + time.sleep(2) + raise Failed(f"Gitea kept #{pr['number']} from merging") + + def _github_auto_merge(self, pr: dict, method: str) -> None: + """Enable auto-merge, or merge when GitHub says the pull request + is in clean status, as hercules-ci-effects does.""" + if self.url == "https://github.com": + graphql = "https://api.github.com/graphql" + else: + graphql = f"{self.url}/api/graphql" + query = ( + "mutation ($id: ID!, $method: PullRequestMergeMethod!) {" + " enablePullRequestAutoMerge(input: {pullRequestId: $id," + " mergeMethod: $method}) { clientMutationId } }" + ) + variables = {"id": pr["node_id"], "method": method.upper()} + reply = self._send(graphql, {"query": query, "variables": variables}) + errors = reply.get("errors") or [] + if not errors: + print(f"Enabled auto-merge ({method}) on #{pr['number']}") + elif any("clean status" in e.get("message", "") for e in errors): + print(f"#{pr['number']} is already in clean status. Merging.") + url = f"{self._github_api()}/repos/{self.path}/pulls/{pr['number']}/merge" + self._send(url, {"merge_method": method}, method="PUT") + else: + raise Failed( + f"enabling auto-merge failed: {errors}\nAuto-merge must " + "be allowed in the repository settings, and the base " + "branch needs a protection rule with required checks." + ) + + +def nix_flake_update(nix: str, path: str, inputs: list[str], summary: str) -> None: + command = [ + nix, + "--extra-experimental-features", + "nix-command flakes", + "flake", + "update", + *inputs, + "--commit-lock-file", + ] + if summary: + command += ["--commit-lockfile-summary", summary] + # Answer "n" to accepting nixConfig, as hercules-ci-effects does. + subprocess.run(command, cwd=path, input="n\n" * 100, text=True, check=True) + + +def flake_inputs(rev: str, path: str, flake: dict) -> list[str]: + """The inputs to update: those named, or the lock file's direct + inputs, less the excluded ones.""" + inputs = flake.get("inputs") or [] + excluded = flake.get("excludeInputs") or [] + if not inputs and (excluded or rev != "HEAD"): + lock = Path(path, "flake.lock").as_posix() + if not has_ref(f"{rev}:{lock}"): + print(f"Skipping {path}: it has no flake.lock", file=sys.stderr) + return [] + locks = json.loads(git("show", f"{rev}:{lock}")) + inputs = sorted(locks["nodes"][locks["root"]].get("inputs", {})) + return [i for i in inputs if i not in excluded] + + +def identity(rev: str) -> str: + """A commit, less its dates and people.""" + return git("log", "-1", "--format=%T%n%P%n%B", rev) + + +def merge_base(update_branch: str, base: str, method: str | None) -> None: + """Check out the update branch: brought up to date with the base by + `method`, or as it stands without one. A new branch, or one that is + reset, starts from the base.""" + update = f"refs/remotes/origin/{update_branch}" + if not has_ref(update) or method == "reset": + git("checkout", "--quiet", "-B", update_branch, base) + return + git("checkout", "--quiet", "-B", update_branch, update) + if method is None: + return + command = { + "merge": ["merge", "--no-edit", base], + "rebase": ["rebase", base], + "fast-forward": ["merge", "--ff-only", base], + }[method] + if subprocess.run(["git", *command], check=False).returncode == 0: + return + if method == "fast-forward": + raise Failed( + f"{update_branch} has commits the base lacks, so it cannot " + f"fast-forward. Delete it (git push origin :{update_branch})" + " or merge it, or use another baseMergeMethod." + ) + conflicts = git("diff", "--name-only", "--diff-filter=U") + raise Failed( + f"Resolve the conflicts by hand and push to {update_branch}:\n{conflicts}" + ) + + +def push(update_branch: str, before: str, force: str | None) -> bool: + """Push the update branch if the update committed anything.""" + if git("diff", "HEAD", "--stat"): + raise Failed("the update left uncommitted changes") + if git("rev-parse", "HEAD") == before: + print(f"No updates to push to {update_branch}.") + return False + # A rebuilt branch differs from the pushed one only in its dates + # unless an input moved: pushing it would rerun CI for nothing. + remote = f"refs/remotes/origin/{update_branch}" + if has_ref(remote) and identity("HEAD") == identity(remote): + print(f"{update_branch} is up to date.") + return True + flags = [f"--{force}"] if force else [] + git("push", "origin", f"HEAD:refs/heads/{update_branch}", *flags) + return True + + +def open_pull_request( + forge: Forge, config: dict, update_branch: str, base: str, title: str +) -> None: + update = f"refs/remotes/origin/{update_branch}" + # Too many pull requests beats too few: make sure one is open. + if not config["createPullRequest"] or not has_ref(update): + return + body = config["pullRequestBody"] + if body is None: + body = git("log", "-1", "--format=%b", update) + pr = forge.open_pull_request(update_branch, base, title, body) + method = config["autoMergeMethod"] + if method is None: + return + # A push by someone else turns GitHub's auto-merge off again. + pr = pr or forge.find_open_pull_request(update_branch) + if pr is not None: + forge.auto_merge(pr, method) + + +def title_for(paths: list[str]) -> str: + """As hercules-ci-effects names it.""" + names = ["`flake.lock`" if p == "." else f"`{p}/flake.lock`" for p in paths] + return "`flake.lock`" if len(names) > 3 else ", ".join(names) + + +def update_together(forge: Forge, config: dict, default_branch: str, nix: str) -> None: + """One branch and pull request for every flake, as upstream.""" + branch = config["updateBranch"] + base_branch = config["baseMergeBranch"] + # Upstream's baseMerge.enable: a method applies only with a branch. + method = config["baseMergeMethod"] if base_branch else None + merge_base(branch, f"refs/remotes/origin/{base_branch or default_branch}", method) + before = git("rev-parse", "HEAD") + flakes = config["flakes"] + for path, flake in flakes.items(): + inputs = flake_inputs("HEAD", path, flake) + if not inputs and (flake.get("inputs") or flake.get("excludeInputs")): + continue + where = "" if list(flakes) == ["."] else f" in '{path}'" + print(f"Running nix flake update{where}...", file=sys.stderr) + nix_flake_update(nix, path, inputs, flake.get("commitSummary", "")) + # Rebase rewrites the branch, and reset starts it over. + force = {"rebase": "force-with-lease", "reset": "force"}.get(method or "") + push(branch, before, force) + title = config["pullRequestTitle"] or f"{title_for(list(flakes))}: Update" + open_pull_request(forge, config, branch, default_branch, title) + + +def update_each(forge: Forge, config: dict, default_branch: str, nix: str) -> None: + """A branch and pull request per input, each started afresh from the + base. One failing input does not stop the others.""" + base = f"refs/remotes/origin/{config['baseMergeBranch'] or default_branch}" + failed = [] + for path, flake in config["flakes"].items(): + inputs = flake_inputs(base, path, flake) + where = "" if path == "." else f" in {path}" + for name in inputs: + suffix = "" if path == "." else f"-{path}" + branch = f"{config['updateBranch']}{suffix}-{name}".replace("/", "-") + summary = f"Update {name}{where}" + try: + print(f"{summary}...", file=sys.stderr) + git("checkout", "--quiet", "-B", branch, base) + before = git("rev-parse", "HEAD") + nix_flake_update(nix, path, [name], summary) + if not push(branch, before, "force") and has_ref( + f"refs/remotes/origin/{branch}" + ): + # The base has the update already. + git("push", "origin", f":refs/heads/{branch}") + continue + open_pull_request( + forge, + config, + branch, + default_branch, + config["pullRequestTitle"] or summary, + ) + except ERRORS as e: + print(f"{summary} failed: {e}", file=sys.stderr) + failed.append(summary) + if failed: + raise Failed("failed: " + ", ".join(failed)) + + +def run(config: dict) -> None: + for who in ("AUTHOR", "COMMITTER"): + os.environ.setdefault(f"GIT_{who}_NAME", "nixbot") + os.environ.setdefault(f"GIT_{who}_EMAIL", "nixbot@localhost") + forge = Forge(config) + forge.setup_git() + head = git("ls-remote", "--symref", "origin", "HEAD") + default_branch = head.split("\t", 1)[0].removeprefix("ref: refs/heads/") + # The checkout comes from nixbot's mirror. The forge has the + # branches as they are now. + git("fetch", "--quiet", "origin", "+refs/heads/*:refs/remotes/origin/*") + if config["perInput"]: + update_each(forge, config, default_branch, config["nix"]) + else: + update_together(forge, config, default_branch, config["nix"]) + + +def main() -> None: + with open(sys.argv[1]) as f: + config = json.load(f) + try: + run(config) + except ERRORS as e: + sys.exit(f"flakeUpdate: {e}") + + +main()