From e5a482789602478bba7d29d92ceb67bb05c0845e Mon Sep 17 00:00:00 2001 From: josie Date: Mon, 18 May 2026 16:45:55 +0200 Subject: [PATCH 1/8] feat: add wireguard-mesh module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A thin wrapper around `networking.wireguard.interfaces` that takes a mesh-wide `peers` definition (same on every host) and builds the interface from it, dropping the entry that matches `thisHost`. Adding a new node is a one-place edit to `peers` plus `thisHost` on the new box. Mesh is point-to-point with /32 peer allowedIPs — no subnets routed through. Exposure of services on the mesh interface is the consumer's concern (scope via `networking.firewall.interfaces.`). Includes a two-VM nixosTest (`checks..wireguard-mesh`) that brings up the mesh on a shared subnet, asserts cross-mesh reachability, and confirms the firewall opens the WG UDP port. Co-Authored-By: Claude Opus 4.7 (1M context) --- flake.nix | 18 ++++ modules/wireguard-mesh.nix | 182 +++++++++++++++++++++++++++++++++++++ test/mesh.nix | 104 +++++++++++++++++++++ 3 files changed, 304 insertions(+) create mode 100644 modules/wireguard-mesh.nix create mode 100644 test/mesh.nix diff --git a/flake.nix b/flake.nix index 78abe0e..4c5376e 100644 --- a/flake.nix +++ b/flake.nix @@ -56,6 +56,7 @@ frigate = ./modules/frigate.nix; hetzner-bare-metal = ./modules/presets/hetzner-bare-metal.nix; public-frigate = ./modules/presets/public-frigate.nix; + wireguard-mesh = ./modules/wireguard-mesh.nix; # Batteries-included entry point. Bundles nix-bitcoin so the # consumer needs only `roost` in their flake inputs to deploy a @@ -102,6 +103,20 @@ inherit pkgs extraModules; roost = self; }; + + # Two-node test of the wireguard-mesh module. Boots two VMs on + # the test driver's shared virtual network, brings up the mesh, + # and verifies cross-mesh reachability + firewall scoping. + mkMeshTest = + { + pkgs, + extraModules ? [ ], + }: + import ./test/mesh.nix { + inherit pkgs extraModules; + roost = self; + }; + }; checks = forAllLinux (system: { @@ -112,6 +127,9 @@ regtest-preset = self.lib.mkRegtestPresetE2E { pkgs = pkgsFor system; }; + wireguard-mesh = self.lib.mkMeshTest { + pkgs = pkgsFor system; + }; }); templates.default = { diff --git a/modules/wireguard-mesh.nix b/modules/wireguard-mesh.nix new file mode 100644 index 0000000..11065dd --- /dev/null +++ b/modules/wireguard-mesh.nix @@ -0,0 +1,182 @@ +{ + config, + lib, + ... +}: + +# Thin WireGuard-mesh wrapper around `networking.wireguard.interfaces`. +# +# The same `peers` block is intended to be defined identically on every +# member host; only `thisHost` and `privateKeyFile` differ per node. The +# module enumerates the peer set, drops the entry matching `thisHost`, +# and emits a wireguard peer for each remainder. Adding a third node +# becomes a one-place edit: a new entry in `peers` plus `thisHost` on +# the new host. +# +# The mesh is point-to-point with /32 peer allowedIPs — no subnets get +# routed through. Public exposure (mesh interface ↔ consumer services) +# is the consumer's concern (scope via `networking.firewall.interfaces.`). + +let + cfg = config.services.roost.wireguard-mesh; + + # CIDR like "10.42.0.0/24" -> prefix length "24". Falls back to "32" + # if the input isn't parseable, which the assertion below catches. + cidrParts = builtins.match "([0-9.]+)/([0-9]+)" cfg.meshCidr; + cidrPrefixLen = if cidrParts == null then "32" else builtins.elemAt cidrParts 1; + + thisPeer = cfg.peers.${cfg.thisHost} or null; + interfaceAddress = lib.optionalString (thisPeer != null) "${thisPeer.meshIp}/${cidrPrefixLen}"; + + otherPeers = lib.filterAttrs (name: _: name != cfg.thisHost) cfg.peers; + + toWgPeer = + peer: + { + publicKey = peer.publicKey; + endpoint = peer.endpoint; + allowedIPs = [ "${peer.meshIp}/32" ]; + } + // lib.optionalAttrs (peer.persistentKeepalive != null) { + inherit (peer) persistentKeepalive; + }; +in +{ + options.services.roost.wireguard-mesh = with lib; { + enable = mkEnableOption "WireGuard mesh between roost hosts"; + + interface = mkOption { + type = types.str; + default = "wg0"; + description = '' + Name of the wireguard interface to create. Override if `wg0` is + already in use on the host for another purpose. + ''; + }; + + thisHost = mkOption { + type = types.str; + description = '' + Short name of the current host within the mesh. Must be a key of + `peers`. The mesh IP for this host is `peers..meshIp`. + ''; + }; + + privateKeyFile = mkOption { + type = types.path; + description = '' + Path to this host's WireGuard private key (typically an + agenix-decrypted path under /run/agenix/). The file must be + readable by root and contain a single base64-encoded key as + produced by `wg genkey`. + ''; + }; + + port = mkOption { + type = types.port; + default = 51820; + description = "UDP port WireGuard listens on. Opened in the firewall."; + }; + + meshCidr = mkOption { + type = types.str; + example = "10.42.0.0/24"; + description = '' + CIDR covering every `peers.*.meshIp`. Only the prefix length is + used (to size the wireguard interface address); the network + portion is informational and documented for operators. + ''; + }; + + mtu = mkOption { + type = types.nullOr types.int; + default = null; + description = '' + Override the WireGuard interface MTU. Null = the upstream + default (1420). Lower this only if the path MTU between mesh + members is below 1500. + ''; + }; + + peers = mkOption { + description = '' + All mesh members keyed by short host name. Define identically + on every member; the module skips the entry matching `thisHost` + when emitting wireguard peers. + ''; + type = types.attrsOf ( + types.submodule { + options = { + publicKey = mkOption { + type = types.str; + description = "Base64 WireGuard public key as produced by `wg pubkey`."; + }; + endpoint = mkOption { + type = types.str; + example = "1.2.3.4:51820"; + description = '' + Public reachable endpoint of this peer (`ip:port` or + `hostname:port`). DNS is resolved once by `wg` at + interface setup — if the address can change, configure + `networking.wireguard.dynamicEndpointRefreshSeconds` at + the consumer level. + ''; + }; + meshIp = mkOption { + type = types.str; + example = "10.42.0.1"; + description = "Mesh-side IPv4 address for this peer. Must fall inside `meshCidr`."; + }; + persistentKeepalive = mkOption { + type = types.nullOr types.int; + default = 25; + description = '' + Seconds between keepalive packets to this peer. 25 is the + conventional "always-on" value — harmless on bare-metal + links and useful behind NAT or any stateful middlebox. + Null disables keepalives. + ''; + }; + }; + } + ); + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.peers ? ${cfg.thisHost}; + message = '' + services.roost.wireguard-mesh.thisHost ("${cfg.thisHost}") must name + an entry in services.roost.wireguard-mesh.peers. Existing peers: + ${lib.concatStringsSep ", " (lib.attrNames cfg.peers)}. + ''; + } + { + assertion = cidrParts != null; + message = '' + services.roost.wireguard-mesh.meshCidr ("${cfg.meshCidr}") is not a + valid IPv4 CIDR. Expected form: "10.42.0.0/24". + ''; + } + { + assertion = (builtins.length (lib.attrNames cfg.peers)) >= 2; + message = '' + services.roost.wireguard-mesh.peers must have at least two members + (this host + at least one remote). A single-node mesh is a no-op. + ''; + } + ]; + + networking.wireguard.interfaces.${cfg.interface} = { + ips = [ interfaceAddress ]; + listenPort = cfg.port; + privateKeyFile = toString cfg.privateKeyFile; + mtu = cfg.mtu; + peers = lib.mapAttrsToList (_name: toWgPeer) otherPeers; + }; + + networking.firewall.allowedUDPPorts = [ cfg.port ]; + }; +} diff --git a/test/mesh.nix b/test/mesh.nix new file mode 100644 index 0000000..6b87efd --- /dev/null +++ b/test/mesh.nix @@ -0,0 +1,104 @@ +{ + pkgs, + roost, + extraModules ? [ ], +}: + +# Two-node nixosTest for the `wireguard-mesh` module. Both VMs sit on +# the same virtual network; the mesh interface is layered on top, with +# /32 allowedIPs scoping the peer relationships. The test asserts that +# mesh IPs reach each other, that the firewall opens the WireGuard UDP +# port automatically, and that the assertions catch a misconfigured +# `thisHost`. +# +# Keypairs below are throwaway test fixtures generated specifically for +# this file. They have no relationship to production hosts and are +# committed deliberately so the test stays pure (no IFD). +let + testKeys = { + a = { + privateKey = "AF3qED26m1FhgY3yn7gvBKP76qPcKoej0oTVaetMZkU="; + publicKey = "PRbUI7dXfSREqCH9twFOaugCW5OrTl2T4RU55F6YGHU="; + }; + b = { + privateKey = "MAN5lxJ4l3bTug+rxk7YMhmIhoPy/13BspwvLnJHUVw="; + publicKey = "vufhiWpCvP7C8LpG9WjXqJk78KJUYDGHcl5Wn3I2xSU="; + }; + }; + + # The wireguard module wants a path on disk, not a literal key. Drop + # each test key into the nix store and reference it by path. Mode 600 + # matches what agenix would produce. + privFor = + name: + pkgs.writeTextFile { + name = "wg-mesh-test-${name}.priv"; + text = testKeys.${name}.privateKey; + }; + + meshPeers = { + a = { + publicKey = testKeys.a.publicKey; + # `nodes..networking.primaryIPAddress` is the canonical way + # to reference a VM's primary NIC address inside a nixosTest; the + # test driver wires nodes onto a shared subnet at runtime. We + # cannot reference that inside `nodes.*` (cyclic), so hardcode + # the default test driver addresses here: `nodeA` gets .2, `nodeB` + # gets .3 on 192.168.1.0/24. + endpoint = "192.168.1.2:51820"; + meshIp = "10.42.0.1"; + }; + b = { + publicKey = testKeys.b.publicKey; + endpoint = "192.168.1.3:51820"; + meshIp = "10.42.0.2"; + }; + }; + + mkNode = name: { + imports = [ + roost.nixosModules.wireguard-mesh + ] + ++ extraModules; + + services.roost.wireguard-mesh = { + enable = true; + thisHost = name; + privateKeyFile = privFor name; + meshCidr = "10.42.0.0/24"; + peers = meshPeers; + }; + }; +in +pkgs.testers.runNixOSTest { + name = "wireguard-mesh"; + + nodes.nodeA = mkNode "a"; + nodes.nodeB = mkNode "b"; + + testScript = '' + start_all() + + # WireGuard runs as a wg-quick-style systemd target generated by the + # `networking.wireguard.interfaces` module. Wait for the unit each + # node owns before pinging across the mesh. + nodeA.wait_for_unit("wireguard-wg0.service") + nodeB.wait_for_unit("wireguard-wg0.service") + + # The interface should be up with the configured /24 address. The + # `ip addr show` output includes "10.42.0.1/24" for nodeA only. + nodeA.succeed("ip -4 addr show wg0 | grep -q 10.42.0.1/24") + nodeB.succeed("ip -4 addr show wg0 | grep -q 10.42.0.2/24") + + # Cross-mesh reachability. The first handshake can take a moment + # after both ends finish their boot, so allow a few attempts. + nodeA.wait_until_succeeds("ping -c 1 -W 1 10.42.0.2", timeout=30) + nodeB.wait_until_succeeds("ping -c 1 -W 1 10.42.0.1", timeout=30) + + # Firewall should have UDP 51820 open. Confirm by inspecting the + # iptables ruleset rather than poking the port from outside, which + # would race against the in-progress handshake. + nodeA.succeed("iptables-save | grep -E -- '--dport 51820'") + nodeB.succeed("iptables-save | grep -E -- '--dport 51820'") + ''; +} From e4c73698f3e4b91bc734cd6fe21b46851cda2882 Mon Sep 17 00:00:00 2001 From: josie Date: Mon, 18 May 2026 16:47:08 +0200 Subject: [PATCH 2/8] feat: split public-frigate + frigate-edge preset for multi-box Adds a frigate-edge preset for the multi-box deployment shape: TLS + ACME + frigate, with bitcoind/fulcrum/ZMQ on another host. Edge nodes authenticate to bitcoind via USERPASS (cookie auth doesn't cross host boundaries); the credentials file is consumed via systemd's LoadCredential and templated into config.toml at start. Refactor: the ACME/nginx/TLS wiring shared between public-frigate and frigate-edge moves into a private `_internal/frigate-tls-acme.nix` helper, set up by the parent preset via the `services._roost.*` internal namespace. No behavior change for existing public-frigate consumers; the regtest-preset test still passes the same assertions. New `exposeBackends` option block on public-frigate lets a backend host bind its bitcoind RPC, ZMQ sequence publisher, and fulcrum on a mesh interface in addition to loopback, with firewall rules scoped to that interface only. Backed by the existing nix-bitcoin typed options where they support it (rpc.users, rpc.allowip), `extraConfig` where they're single-bind (rpcbind, fulcrum tcp). Test: `checks..regtest-edge` boots two VMs (backend running the full local stack with exposeBackends on; edge running frigate-edge against it) and runs the same scan-end-to-end checks regtest-preset runs, driven against the edge's Electrum listener. Co-Authored-By: Claude Opus 4.7 (1M context) --- flake.nix | 18 ++ modules/_internal/frigate-tls-acme.nix | 145 ++++++++++++++++ modules/presets/frigate-edge.nix | 161 +++++++++++++++++ modules/presets/public-frigate.nix | 229 +++++++++++++++---------- test/regtest-edge.nix | 219 +++++++++++++++++++++++ 5 files changed, 680 insertions(+), 92 deletions(-) create mode 100644 modules/_internal/frigate-tls-acme.nix create mode 100644 modules/presets/frigate-edge.nix create mode 100644 test/regtest-edge.nix diff --git a/flake.nix b/flake.nix index 4c5376e..9c5ce3b 100644 --- a/flake.nix +++ b/flake.nix @@ -56,6 +56,7 @@ frigate = ./modules/frigate.nix; hetzner-bare-metal = ./modules/presets/hetzner-bare-metal.nix; public-frigate = ./modules/presets/public-frigate.nix; + frigate-edge = ./modules/presets/frigate-edge.nix; wireguard-mesh = ./modules/wireguard-mesh.nix; # Batteries-included entry point. Bundles nix-bitcoin so the @@ -117,6 +118,20 @@ roost = self; }; + # Two-VM end-to-end test for the frigate-edge preset. Boots a + # full nix-bitcoin stack on the `backend` node (with the + # public-frigate exposeBackends option enabled) and a slim + # frigate-edge consumer on the `edge` node, then exercises the + # edge's Electrum listeners. + mkRegtestEdgeE2E = + { + pkgs, + extraModules ? [ ], + }: + import ./test/regtest-edge.nix { + inherit pkgs extraModules; + roost = self; + }; }; checks = forAllLinux (system: { @@ -127,6 +142,9 @@ regtest-preset = self.lib.mkRegtestPresetE2E { pkgs = pkgsFor system; }; + regtest-edge = self.lib.mkRegtestEdgeE2E { + pkgs = pkgsFor system; + }; wireguard-mesh = self.lib.mkMeshTest { pkgs = pkgsFor system; }; diff --git a/modules/_internal/frigate-tls-acme.nix b/modules/_internal/frigate-tls-acme.nix new file mode 100644 index 0000000..7504fb7 --- /dev/null +++ b/modules/_internal/frigate-tls-acme.nix @@ -0,0 +1,145 @@ +{ + config, + lib, + pkgs, + ... +}: + +# Internal helper: TLS + ACME wiring shared between the `public-frigate` +# and `frigate-edge` presets. Not exported via `nixosModules` and not +# part of the stable API — the options below are flagged `internal`. +# +# A parent preset enables this module and feeds it `host` + `tls`. The +# module materializes `services.frigate.sslCert` / `sslKey`, ACME via +# webroot when an email is set, the nginx vhost serving the HTTP-01 +# challenge, the PKCS#8 key conversion frigate's TLS loader requires, +# and the systemd ordering that prevents frigate from racing the +# initial cert issuance. + +let + cfg = config.services._roost.frigate-tls-acme; + + certFile = + if cfg.tls.certificateFile != null then + cfg.tls.certificateFile + else + "/var/lib/acme/${cfg.host}/fullchain.pem"; + + keyFile = if cfg.tls.keyFile != null then cfg.tls.keyFile else "/var/lib/acme/${cfg.host}/key.pem"; +in +{ + options.services._roost.frigate-tls-acme = with lib; { + enable = mkOption { + type = types.bool; + default = false; + internal = true; + description = "Enable shared TLS + ACME wiring. Set by a parent preset, not by hand."; + }; + + host = mkOption { + type = types.str; + internal = true; + }; + + tls = { + acmeEmail = mkOption { + type = types.nullOr types.str; + default = null; + internal = true; + }; + certificateFile = mkOption { + type = types.nullOr types.path; + default = null; + internal = true; + }; + keyFile = mkOption { + type = types.nullOr types.path; + default = null; + internal = true; + }; + }; + }; + + config = lib.mkIf cfg.enable ( + lib.mkMerge [ + { + assertions = [ + { + assertion = + (cfg.tls.acmeEmail == null) || (cfg.tls.certificateFile == null && cfg.tls.keyFile == null); + message = '' + tls.acmeEmail is mutually exclusive with tls.certificateFile / tls.keyFile. + ''; + } + { + assertion = + (cfg.tls.acmeEmail != null) || (cfg.tls.certificateFile != null && cfg.tls.keyFile != null); + message = '' + TLS requires either tls.acmeEmail (ACME-issued) or both tls.certificateFile + and tls.keyFile (operator-managed). + ''; + } + ]; + + services.frigate.sslCert = certFile; + services.frigate.sslKey = keyFile; + services.frigate.extraSupplementaryGroups = lib.optional (cfg.tls.acmeEmail != null) "acme"; + } + + (lib.mkIf (cfg.tls.acmeEmail != null) { + security.acme = { + acceptTerms = true; + defaults.email = cfg.tls.acmeEmail; + }; + + # Manage the cert directly via `webroot` HTTP-01 rather than + # nginx's `enableACME` shorthand. The shorthand auto-registers + # nginx (and `nginx-config-reload.service` as root) as cert + # consumers and adds an assertion that the cert be readable by + # both — but our cert lives in the `acme` group for frigate, + # and neither nginx nor the reload service joins it. nginx + # here only needs to serve the HTTP-01 challenge files lego + # drops into the webroot; it never touches the issued cert. + # + # postRun: frigate's TLS loader only accepts PKCS#8 + # (`BEGIN PRIVATE KEY`), but lego emits EC keys in SEC1 + # (`BEGIN EC PRIVATE KEY`) and RSA keys in PKCS#1 + # (`BEGIN RSA PRIVATE KEY`). Convert key.pem in place after + # each issuance/renewal so frigate can parse it. Runs as root + # in the cert directory; `chown acme:acme` keeps the file + # owned the way NixOS would have set it. Idempotent — running + # `openssl pkcs8 -topk8` on an already-PKCS#8 key is a no-op. + security.acme.certs.${cfg.host} = { + domain = cfg.host; + webroot = "/var/lib/acme/acme-challenge"; + group = "acme"; + reloadServices = [ "frigate.service" ]; + postRun = '' + umask 0027 + ${pkgs.openssl}/bin/openssl pkcs8 -topk8 -nocrypt \ + -in key.pem -out key.pem.pkcs8 + chown acme:acme key.pem.pkcs8 + mv key.pem.pkcs8 key.pem + ''; + }; + + services.nginx = { + enable = true; + virtualHosts.${cfg.host} = { + locations."/.well-known/acme-challenge/".root = "/var/lib/acme/acme-challenge"; + locations."/".return = "404"; + }; + }; + + networking.firewall.allowedTCPPorts = [ 80 ]; + + # Block frigate startup until the cert exists, otherwise it + # crash-loops on a missing `fullchain.pem` during a fresh + # deploy. `wants` (not `requires`) so a transient acme failure + # later does not take frigate down with it. + systemd.services.frigate.after = [ "acme-${cfg.host}.service" ]; + systemd.services.frigate.wants = [ "acme-${cfg.host}.service" ]; + }) + ] + ); +} diff --git a/modules/presets/frigate-edge.nix b/modules/presets/frigate-edge.nix new file mode 100644 index 0000000..899a193 --- /dev/null +++ b/modules/presets/frigate-edge.nix @@ -0,0 +1,161 @@ +{ + config, + lib, + ... +}: + +# Edge-mode Frigate: TLS + ACME + frigate, with bitcoind and fulcrum +# living on another host. The consumer points `backend.bitcoind.rpcUrl`, +# `backend.bitcoind.zmqSequenceEndpoint`, and `backend.electrumUrl` at +# the remote endpoints — typically over a private WireGuard mesh (see +# `roost.nixosModules.wireguard-mesh`) — and supplies a credentials +# file containing `user:password` for the bitcoind RPC. +# +# This preset is intentionally narrow: no nix-bitcoin, no local +# services.bitcoind or services.fulcrum, no `manage` flags. If you want +# everything on one box, use `public-frigate` (or `nixosModules.default`) +# instead. + +let + cfg = config.services.frigate-edge; +in +{ + imports = [ + ../frigate.nix + ../_internal/frigate-tls-acme.nix + ]; + + options.services.frigate-edge = with lib; { + enable = mkEnableOption "edge-mode public Frigate (TLS + ACME, backends on another host)"; + + host = mkOption { + type = types.str; + example = "albatross.example.com"; + description = '' + Public DNS name for this frigate node. Advertised in the Electrum + `server.features` response, used as the SAN clients validate + against the served TLS certificate, and — when `tls.acmeEmail` + is set — as the `security.acme.certs.` identifier. + ''; + }; + + network = mkOption { + type = types.enum [ + "mainnet" + "testnet" + "testnet4" + "signet" + "regtest" + ]; + default = "mainnet"; + }; + + publicPort = mkOption { + type = types.port; + default = 50002; + description = '' + Public TLS port. 50002 is the convention for Electrum-over-SSL. + ''; + }; + + tls = { + acmeEmail = mkOption { + type = types.nullOr types.str; + default = null; + example = "ops@example.com"; + description = '' + Email address for Let's Encrypt registration. Setting it enables + ACME for `host`. Mutually exclusive with manual cert/key files. + ''; + }; + + certificateFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Path to a TLS certificate. Required when not using ACME."; + }; + + keyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Path to the matching PKCS#8 TLS private key. Required when not using ACME."; + }; + }; + + backend = { + bitcoind = { + rpcUrl = mkOption { + type = types.str; + example = "http://10.42.0.1:8332"; + description = '' + URL of the bitcoind JSON-RPC endpoint on the backend host. + Plain `http://` is fine when the transport is a private + mesh; do not expose the backend RPC to the public internet. + ''; + }; + + authCredentialFile = mkOption { + type = types.path; + description = '' + File on disk containing literally `user:password` for the + bitcoind RPC user. Loaded via systemd `LoadCredential` and + substituted into frigate's config.toml at service start; + never read by the frigate process directly. Typically an + agenix-decrypted path under `/run/agenix/`. + + The corresponding rpcauth line (`user:salt$hash`) lives on + the backend host's bitcoin.conf. Generate the pair once via + bitcoind's `rpcauth.py`. + ''; + }; + + zmqSequenceEndpoint = mkOption { + type = types.str; + example = "tcp://10.42.0.1:28336"; + description = '' + URL of the bitcoind ZMQ `sequence` publisher on the backend + host. Frigate subscribes for sub-100ms mempool ingestion. + ''; + }; + }; + + electrumUrl = mkOption { + type = types.str; + example = "tcp://10.42.0.1:60001"; + description = '' + URL of the backing Electrum server (fulcrum) on the backend + host. Frigate proxies non-silent-payments queries here. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + # TLS + ACME wiring is shared with public-frigate; delegate to the + # private helper module. TLS-mutex assertions live there. + services._roost.frigate-tls-acme = { + enable = true; + inherit (cfg) host tls; + }; + + services.frigate = { + enable = true; + host = cfg.host; + network = cfg.network; + # Plaintext listener stays on loopback. All public traffic + # arrives via the TLS listener below. + tcp = "tcp://127.0.0.1:50001"; + ssl = "ssl://0.0.0.0:${toString cfg.publicPort}"; + bitcoind = { + enable = true; + server = cfg.backend.bitcoind.rpcUrl; + authType = "USERPASS"; + authCredentialFile = cfg.backend.bitcoind.authCredentialFile; + zmqSequenceEndpoint = cfg.backend.bitcoind.zmqSequenceEndpoint; + }; + electrumBackend = cfg.backend.electrumUrl; + }; + + networking.firewall.allowedTCPPorts = [ cfg.publicPort ]; + }; +} diff --git a/modules/presets/public-frigate.nix b/modules/presets/public-frigate.nix index 23d8fba..120e4fb 100644 --- a/modules/presets/public-frigate.nix +++ b/modules/presets/public-frigate.nix @@ -15,22 +15,25 @@ let # `electrumBackend` URL can't drift apart. backendPort = 60001; - # ZMQ `sequence` publisher endpoint. Bitcoin Core opens the socket - # (via `zmqpubsequence=...`) and Frigate subscribes to it (via - # `core.zmqSequenceEndpoint`). Both sides must match exactly. + # The local frigate process always reads ZMQ off loopback; that's a + # constant. When `exposeBackends` is on, bitcoind additionally binds + # the same socket on the mesh address so edge consumers can subscribe + # — see the publish endpoint below. zmqSequenceEndpoint = "tcp://127.0.0.1:28336"; - # When ACME issues the cert, the files live under `/var/lib/acme//`. - # When the consumer brings their own, we point straight at their files. - certFile = - if cfg.tls.certificateFile != null then - cfg.tls.certificateFile - else - "/var/lib/acme/${cfg.host}/fullchain.pem"; - keyFile = if cfg.tls.keyFile != null then cfg.tls.keyFile else "/var/lib/acme/${cfg.host}/key.pem"; + # Where bitcoind opens the ZMQ socket. With no edge consumers, bind + # to loopback only. With `exposeBackends.enable`, bind to 0.0.0.0 so + # both local frigate (via 127.0.0.1) and remote edge frigate (via + # `bindAddress`) can subscribe; the firewall scopes outside access + # to `exposeBackends.interface` only. + zmqPublishBind = if cfg.exposeBackends.enable then "0.0.0.0" else "127.0.0.1"; + zmqPublishEndpoint = "tcp://${zmqPublishBind}:28336"; in { - imports = [ ../frigate.nix ]; + imports = [ + ../frigate.nix + ../_internal/frigate-tls-acme.nix + ]; options.services.public-frigate = with lib; { enable = mkEnableOption "public-facing Frigate silent payments server"; @@ -113,6 +116,61 @@ in ''; }; + exposeBackends = { + enable = mkEnableOption "expose bitcoind RPC/ZMQ and fulcrum for edge consumers"; + + bindAddress = mkOption { + type = types.str; + example = "10.42.0.1"; + description = '' + Additional address bitcoind RPC, ZMQ sequence, and fulcrum + bind to (in addition to their loopback defaults). Typically + this host's mesh IP — see `roost.nixosModules.wireguard-mesh`. + ''; + }; + + interface = mkOption { + type = types.str; + example = "wg0"; + description = '' + Interface name used to scope the firewall rules that open the + backend ports. Only traffic arriving on this interface is + accepted; the backends remain unreachable from the public + internet. + ''; + }; + + allowedPeers = mkOption { + type = types.listOf types.str; + example = [ "10.42.0.2/32" ]; + description = '' + Source CIDRs added to bitcoind's `rpcallowip`. Must include + every edge consumer's mesh IP (/32) that needs to talk to the + backends. Loopback is always allowed. + ''; + }; + + rpcAuth = { + user = mkOption { + type = types.str; + example = "frigate-edge"; + description = "RPC user name added to bitcoind for edge consumers."; + }; + + passwordHMAC = mkOption { + type = types.str; + example = "f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae"; + description = '' + Literal `salt$hash` portion of an rpcauth line, as produced + by bitcoind's `rpcauth.py`. Committed to nix config — the + HMAC is one-way derived from the password; only the + corresponding plaintext is a secret (lives on the edge + consumer). + ''; + }; + }; + }; + # Sentinel attribute, mirroring nix-bitcoin's `secure-node-preset-enabled`. # Lets downstream modules and tests detect activation without re-checking # every individual service. @@ -127,6 +185,15 @@ in lib.mkMerge [ { services.public-frigate.preset-enabled = { }; } + # TLS + ACME wiring is shared with frigate-edge; delegate to the + # private helper module. TLS-mutex assertions live there too. + { + services._roost.frigate-tls-acme = { + enable = true; + inherit (cfg) host tls; + }; + } + { assertions = [ { @@ -150,22 +217,6 @@ in and enable it, or set services.public-frigate.fulcrum.manage = true. ''; } - { - assertion = - (cfg.tls.acmeEmail == null) || (cfg.tls.certificateFile == null && cfg.tls.keyFile == null); - message = '' - services.public-frigate.tls.acmeEmail is mutually exclusive with - tls.certificateFile / tls.keyFile. - ''; - } - { - assertion = - (cfg.tls.acmeEmail != null) || (cfg.tls.certificateFile != null && cfg.tls.keyFile != null); - message = '' - services.public-frigate requires either tls.acmeEmail (for ACME) - or both tls.certificateFile and tls.keyFile (for a manual cert). - ''; - } ]; } @@ -197,14 +248,15 @@ in # all public traffic comes in over `ssl`. The backend Electrum # server (fulcrum/electrs/etc.) listens on a non-conflicting port # so frigate can occupy the canonical Electrum ports. + # + # `sslCert`, `sslKey` and `extraSupplementaryGroups` are set by + # the shared TLS+ACME helper (imported above). services.frigate = { enable = true; host = cfg.host; network = cfg.network; tcp = "tcp://127.0.0.1:50001"; ssl = "ssl://0.0.0.0:${toString cfg.publicPort}"; - sslCert = certFile; - sslKey = keyFile; bitcoind = { enable = true; server = "http://127.0.0.1:8332"; @@ -213,11 +265,6 @@ in inherit zmqSequenceEndpoint; }; electrumBackend = "tcp://127.0.0.1:${toString backendPort}"; - # ACME-issued certs live in /var/lib/acme// owned by the - # `acme` group. Frigate reads them at startup, so its service - # needs the group. Skipped for manual-cert deployments where - # the operator has already arranged read access. - extraSupplementaryGroups = lib.optional (cfg.tls.acmeEmail != null) "acme"; }; users.users.frigate.extraGroups = [ "bitcoin" ]; @@ -260,73 +307,71 @@ in # the nix-bitcoin module already assigns the string. (lib.mkIf cfg.bitcoind.manage { services.bitcoind.extraConfig = '' - zmqpubsequence=${zmqSequenceEndpoint} + zmqpubsequence=${zmqPublishEndpoint} ''; systemd.services.bitcoind.serviceConfig.RestrictAddressFamilies = lib.mkForce "AF_UNIX AF_INET AF_INET6 AF_NETLINK"; }) - # ACME path: a minimal HTTP vhost on port 80 hosts the HTTP-01 - # challenge so Let's Encrypt can verify domain ownership. NixOS's - # `enableACME` wires `security.acme.certs.` and the challenge - # location automatically; the `404` covers anything else hitting - # this vhost. nginx is only here for ACME — TLS termination for - # the Electrum stream is frigate's job. - (lib.mkIf (cfg.tls.acmeEmail != null) { - security.acme = { - acceptTerms = true; - defaults.email = cfg.tls.acmeEmail; - }; - - # Manage the cert directly via `webroot` HTTP-01 rather than - # nginx's `enableACME` shorthand. The shorthand auto-registers - # nginx (and `nginx-config-reload.service` as root) as cert - # consumers and adds an assertion that the cert be readable by - # both — but our cert lives in the `acme` group for frigate, - # and neither nginx nor the reload service joins it. nginx - # here only needs to serve the HTTP-01 challenge files lego - # drops into the webroot; it never touches the issued cert. - # - # postRun: frigate's TLS loader only accepts PKCS#8 - # (`BEGIN PRIVATE KEY`), but lego emits EC keys in SEC1 - # (`BEGIN EC PRIVATE KEY`) and RSA keys in PKCS#1 - # (`BEGIN RSA PRIVATE KEY`). Convert key.pem in place after - # each issuance/renewal so frigate can parse it. Runs as root - # in the cert directory; `chown acme:acme` keeps the file - # owned the way NixOS would have set it. Idempotent — running - # `openssl pkcs8 -topk8` on an already-PKCS#8 key is a no-op. - security.acme.certs.${cfg.host} = { - domain = cfg.host; - webroot = "/var/lib/acme/acme-challenge"; - group = "acme"; - reloadServices = [ "frigate.service" ]; - postRun = '' - umask 0027 - ${pkgs.openssl}/bin/openssl pkcs8 -topk8 -nocrypt \ - -in key.pem -out key.pem.pkcs8 - chown acme:acme key.pem.pkcs8 - mv key.pem.pkcs8 key.pem - ''; - }; + # exposeBackends: bind bitcoind RPC + ZMQ + fulcrum on the mesh + # interface for an edge consumer. Only honored when the preset is + # managing those services locally — exposing services we don't + # manage would be a contract violation. + # + # bitcoind RPC: nix-bitcoin's `rpc.address` is single-valued, so + # we keep the typed loopback default and append a second + # `rpcbind=` via extraConfig. bitcoind accepts repeated rpcbind + # lines and binds each one. + # + # ZMQ: the publish endpoint above (`zmqPublishEndpoint`) already + # flips to 0.0.0.0 when exposeBackends is on — no extraConfig + # work needed here for ZMQ. + # + # fulcrum: same single-bind option pattern as bitcoind RPC. The + # typed `address` stays on loopback; an extra `tcp = ...` line is + # appended via `extraConfig` for the mesh address. + (lib.mkIf cfg.exposeBackends.enable { + assertions = [ + { + assertion = cfg.bitcoind.manage; + message = '' + services.public-frigate.exposeBackends.enable requires + services.public-frigate.bitcoind.manage = true. The preset + cannot expose a bitcoind it does not configure. + ''; + } + { + assertion = cfg.fulcrum.manage; + message = '' + services.public-frigate.exposeBackends.enable requires + services.public-frigate.fulcrum.manage = true. The preset + cannot expose a fulcrum it does not configure. + ''; + } + ]; - services.nginx = { - enable = true; - virtualHosts.${cfg.host} = { - locations."/.well-known/acme-challenge/".root = "/var/lib/acme/acme-challenge"; - locations."/".return = "404"; + services.bitcoind = { + rpc.allowip = [ "127.0.0.1" ] ++ cfg.exposeBackends.allowedPeers; + rpc.users.${cfg.exposeBackends.rpcAuth.user} = { + inherit (cfg.exposeBackends.rpcAuth) passwordHMAC; }; + extraConfig = '' + rpcbind=${cfg.exposeBackends.bindAddress} + ''; }; - networking.firewall.allowedTCPPorts = [ 80 ]; + services.fulcrum.extraConfig = '' + tcp = ${cfg.exposeBackends.bindAddress}:${toString backendPort} + ''; - # Block frigate startup until the cert exists, otherwise it - # crash-loops on a missing `fullchain.pem` during a fresh - # deploy. `wants` (not `requires`) so a transient acme failure - # later doesn't take frigate down with it. List values under - # `systemd.services.` accumulate via module merging, so - # this composes with the bitcoind/fulcrum deps above. - systemd.services.frigate.after = [ "acme-${cfg.host}.service" ]; - systemd.services.frigate.wants = [ "acme-${cfg.host}.service" ]; + # Scope the open ports to the mesh interface only. Outside + # traffic (e.g. the public internet on eth0) is dropped at + # INPUT by NixOS's default-deny firewall posture. + networking.firewall.interfaces.${cfg.exposeBackends.interface}.allowedTCPPorts = [ + 8332 + 28336 + backendPort + ]; }) ] ); diff --git a/test/regtest-edge.nix b/test/regtest-edge.nix new file mode 100644 index 0000000..edaf428 --- /dev/null +++ b/test/regtest-edge.nix @@ -0,0 +1,219 @@ +{ + pkgs, + roost, + extraModules ? [ ], +}: + +# Two-VM end-to-end test for the frigate-edge preset. +# +# backend VM: nix-bitcoin + public-frigate (full local stack) with +# `exposeBackends` enabled so bitcoind RPC/ZMQ and fulcrum +# also listen on the shared subnet for the edge. +# edge VM: frigate-edge consuming the backend's services over the +# shared network. ACME is off (manual cert) so the edge +# boots without DNS or a real CA. +# +# WireGuard is intentionally not in the loop here — that's covered by +# `test/mesh.nix`. This test exists to verify the frigate-edge preset's +# wiring (USERPASS auth, remote ZMQ, remote electrum, ACME bypass via +# manual cert) and the matching `exposeBackends` bind logic on the +# backend side. +# +# The bitcoind RPC password is a fixed test fixture; the rpcauth HMAC +# below was computed from it via `bitcoind/share/rpcauth/rpcauth.py +# frigate-edge testpassword`. Both halves are committed deliberately so +# the test stays pure (no IFD, no out-of-band state). +let + selfSignedCert = + pkgs.runCommand "test-self-signed-cert" + { + nativeBuildInputs = [ pkgs.openssl ]; + } + '' + openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout key.pem -out cert.pem \ + -days 1 -subj "/CN=test.local" + install -d $out + install -m 0644 cert.pem $out/cert.pem + install -m 0644 key.pem $out/key.pem + ''; + + rpcUser = "frigate-edge"; + rpcPassword = "testpassword"; + + # `salt$hash` computed once via HMAC-SHA256(salt, password). Same + # algorithm bitcoind/share/rpcauth/rpcauth.py implements. Committable + # — derives one-way from the plaintext. + rpcPasswordHMAC = "2316d0a5e8ee6339ffb4d86c983bb421$9b90ff10a12e7df0dee2cd86f827461d3a481f1947a0bae613e4046407ee6ced"; + + # `user:password` line the edge feeds frigate via LoadCredential. + authCredentialFile = pkgs.writeText "edge-bitcoind-auth" "${rpcUser}:${rpcPassword}"; + + # Mesh-like IP shared between the two VMs. The nixosTest default + # subnet is 192.168.1.0/24 with the first declared node at .2; pin + # the backend's IP via test-driver options so the edge can address it + # at a known location regardless of order. + backendIp = "192.168.1.2"; +in +pkgs.testers.runNixOSTest { + name = "regtest-edge"; + + nodes.backend = + { + config, + pkgs, + lib, + ... + }: + { + imports = [ + roost.nixosModules.default + ] + ++ extraModules; + + services.public-frigate = { + enable = true; + host = "backend.test.local"; + network = "regtest"; + tls.certificateFile = "${selfSignedCert}/cert.pem"; + tls.keyFile = "${selfSignedCert}/key.pem"; + + exposeBackends = { + enable = true; + bindAddress = backendIp; + interface = "eth1"; + allowedPeers = [ "192.168.1.0/24" ]; + rpcAuth = { + user = rpcUser; + passwordHMAC = rpcPasswordHMAC; + }; + }; + }; + + # Same regtest plumbing as `regtest-preset.nix`. See that file for + # the per-knob rationale. + services.bitcoind = { + regtest = true; + dbCache = lib.mkForce 100; + disablewallet = lib.mkForce false; + extraConfig = '' + maxtipage=2147483647 + ''; + }; + services.frigate.bitcoind.cookieDir = lib.mkForce "/var/lib/bitcoind/regtest"; + services.frigate.computeBackend = lib.mkForce "CPU"; + services.frigate.bitcoind.server = lib.mkForce "http://127.0.0.1:${toString config.services.bitcoind.rpc.port}"; + + networking.firewall.allowedTCPPorts = [ 50001 ]; + + virtualisation.cores = 4; + virtualisation.memorySize = 4096; + }; + + nodes.edge = + { + config, + pkgs, + lib, + ... + }: + { + imports = [ + roost.nixosModules.frigate-edge + ] + ++ extraModules; + + services.frigate-edge = { + enable = true; + host = "edge.test.local"; + network = "regtest"; + tls.certificateFile = "${selfSignedCert}/cert.pem"; + tls.keyFile = "${selfSignedCert}/key.pem"; + + backend = { + bitcoind = { + rpcUrl = "http://${backendIp}:18443"; + inherit authCredentialFile; + zmqSequenceEndpoint = "tcp://${backendIp}:28336"; + }; + electrumUrl = "tcp://${backendIp}:60001"; + }; + }; + + # GPU isn't present in the test VM; pin to CPU compute. Matches + # what regtest-preset does on the backend. + services.frigate.computeBackend = lib.mkForce "CPU"; + + virtualisation.cores = 2; + virtualisation.memorySize = 2048; + }; + + testScript = + { nodes, ... }: + let + cli = "bitcoin-cli -regtest -datadir=/var/lib/bitcoind"; + in + '' + start_all() + + # Backend comes up first; mine the chain so the edge has something + # real to scan. + backend.wait_for_unit("bitcoind.service") + backend.wait_until_succeeds("${cli} getblockchaininfo", timeout=30) + + backend.succeed("${cli} createwallet test") + addr = backend.succeed("${cli} -rpcwallet=test getnewaddress").strip() + backend.succeed(f"${cli} generatetoaddress 101 {addr}") + + backend.wait_until_succeeds( + "${cli} getblockchaininfo | grep -q '\"initialblockdownload\": false'", + timeout=30, + ) + + backend.wait_for_unit("fulcrum.service") + backend.wait_for_open_port(60001, addr="${backendIp}") + + # bitcoind RPC and ZMQ should also be reachable from the second + # interface thanks to exposeBackends. + backend.wait_for_open_port(18443, addr="${backendIp}") + backend.wait_for_open_port(28336, addr="${backendIp}") + + # Edge can talk to the backend via the shared subnet. + edge.wait_until_succeeds("nc -z ${backendIp} 60001", timeout=30) + edge.wait_until_succeeds("nc -z ${backendIp} 18443", timeout=30) + + # Frigate-edge should authenticate against bitcoind (USERPASS, + # plaintext fed via LoadCredential), subscribe to remote ZMQ, and + # accept Electrum traffic on both its plaintext and TLS listeners. + edge.wait_for_unit("frigate.service") + edge.wait_for_open_port(50001) + edge.wait_for_open_port(50002) + + import time + deadline = time.time() + 120 + probe = ( + "{ echo '{\"jsonrpc\":\"2.0\",\"id\":0,\"method\":\"server.version\",\"params\":[\"test\",\"1.4\"]}'" + "; echo '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"server.features\",\"params\":[]}'" + "; echo '{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"blockchain.headers.subscribe\",\"params\":[]}'; }" + " | nc -q 3 127.0.0.1 50001" + ) + internal = "" + while time.time() < deadline: + _status, internal = edge.execute(probe) + print(f"frigate-edge plaintext probe ({len(internal)}B): {internal!r}") + if "edge.test.local" in internal and '"height":101' in internal: + break + time.sleep(2) + else: + raise Exception( + f"frigate-edge plaintext probe never returned expected content. " + f"Last response: {internal!r}" + ) + + assert "edge.test.local" in internal, f"edge server.features missing host: {internal}" + assert '"height":101' in internal, ( + f"edge blockchain.headers.subscribe missing height:101 — fulcrum proxy " + f"or remote backend wiring broken: {internal}" + ) + ''; +} From a133f504d9e651894cdf848f1a9ae42127f5a448 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 12:31:49 +0200 Subject: [PATCH 3/8] fix: wait for wireguard-wg0.target so peers are configured before ping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upstream wireguard module produces two units per interface: a `wireguard-wg0.service` that brings up the interface itself, and one `wireguard-wg0-peer-.service` per peer that installs the peer's config. A target `wireguard-wg0.target` aggregates both. The test was waiting on the bare interface service, which returns as soon as the interface is up — before the per-peer services have installed any peers in the kernel. Pings fired immediately after hit "ping: sendmsg: Required key not available" because there was no peer matching 10.42.0.2 yet, and the 30s timeout expired before the peer service finished its setup. Wait on the target instead. It is `wantedBy = [ "multi-user.target" ]` and `wants` both the interface service and every peer service, so a target-reached state is the right "everything's installed" signal. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/mesh.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/mesh.nix b/test/mesh.nix index 6b87efd..cf99312 100644 --- a/test/mesh.nix +++ b/test/mesh.nix @@ -79,11 +79,13 @@ pkgs.testers.runNixOSTest { testScript = '' start_all() - # WireGuard runs as a wg-quick-style systemd target generated by the - # `networking.wireguard.interfaces` module. Wait for the unit each - # node owns before pinging across the mesh. - nodeA.wait_for_unit("wireguard-wg0.service") - nodeB.wait_for_unit("wireguard-wg0.service") + # The upstream wireguard module creates a target `wireguard-wg0` + # that waits for the interface service AND every peer service. + # Waiting on the bare interface service returns too early — the + # peers aren't in the kernel yet, so ping fails with "Required key + # not available" until the peer services finish. + nodeA.wait_for_unit("wireguard-wg0.target") + nodeB.wait_for_unit("wireguard-wg0.target") # The interface should be up with the configured /24 address. The # `ip addr show` output includes "10.42.0.1/24" for nodeA only. From f9c535400d2fbcdb7cd644c33a0883dcbe016888 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 12:31:53 +0200 Subject: [PATCH 4/8] fix: derive exposeBackends firewall RPC port from config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exposeBackends firewall rule was hardcoding port 8332. That's mainnet's bitcoind RPC port; nix-bitcoin's `rpc.port` default tracks the chain (regtest → 18443, testnet → 18332, signet → 38332). On any non-mainnet network the firewall opens 8332 while bitcoind listens elsewhere, and edge consumers see connection refused. Surfaced by regtest-edge: bitcoind on the backend bound 18443 (regtest), the firewall opened 8332, the edge's frigate hit "Cannot connect to Bitcoin Core at http://192.168.1.2:18443" and the service exited. Pull the port from `config.services.bitcoind.rpc.port` so the firewall follows whatever bitcoind is actually doing. Safe inside this mkIf-block because exposeBackends already asserts bitcoind.manage = true, which guarantees the option is defined. Co-Authored-By: Claude Opus 4.7 (1M context) --- modules/presets/public-frigate.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/presets/public-frigate.nix b/modules/presets/public-frigate.nix index 120e4fb..9a0528a 100644 --- a/modules/presets/public-frigate.nix +++ b/modules/presets/public-frigate.nix @@ -367,8 +367,13 @@ in # Scope the open ports to the mesh interface only. Outside # traffic (e.g. the public internet on eth0) is dropped at # INPUT by NixOS's default-deny firewall posture. + # + # Pull bitcoind's RPC port from config rather than hardcoding + # `8332`. nix-bitcoin's `rpc.port` default tracks the chain + # (8332 mainnet, 18443 regtest, 18332 testnet, etc.), and the + # firewall has to match wherever bitcoind actually listens. networking.firewall.interfaces.${cfg.exposeBackends.interface}.allowedTCPPorts = [ - 8332 + config.services.bitcoind.rpc.port 28336 backendPort ]; From 279983e841485311dfa19362cd1ee7c5875dfd41 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 12:47:00 +0200 Subject: [PATCH 5/8] fix: use correct nixosTest IP allocation in mesh and edge tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NixOS test framework assigns each node's primary interface address as 192.168.., starting at nodeNumber 1 in declaration order (range 1 254 in lib/testing/network.nix:23). I had hardcoded .2 / .3 throughout, which is off-by-one — the first declared node is .1, not .2. Consequence in mesh.nix: nodeA's wireguard peer pointed at 192.168.1.3:51820 (a non-existent IP), nodeB pointed at .2 (which was *its own* address). Handshake never completed; ping timed out with no peer alive. Consequence in regtest-edge.nix: the edge tried to reach the backend at 192.168.1.2:18443, but the backend was actually at .1. Even with the firewall fix in the preceding commit, the edge couldn't find the backend at all. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/mesh.nix | 13 ++++++------- test/regtest-edge.nix | 11 ++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/mesh.nix b/test/mesh.nix index cf99312..1fb5b06 100644 --- a/test/mesh.nix +++ b/test/mesh.nix @@ -40,17 +40,16 @@ let a = { publicKey = testKeys.a.publicKey; # `nodes..networking.primaryIPAddress` is the canonical way - # to reference a VM's primary NIC address inside a nixosTest; the - # test driver wires nodes onto a shared subnet at runtime. We - # cannot reference that inside `nodes.*` (cyclic), so hardcode - # the default test driver addresses here: `nodeA` gets .2, `nodeB` - # gets .3 on 192.168.1.0/24. - endpoint = "192.168.1.2:51820"; + # to reference a VM's primary NIC address inside a nixosTest, but + # we cannot reference that from inside `nodes.*` (cyclic). The + # test framework assigns 192.168.. starting at + # nodeNumber 1, in declaration order: nodeA = .1, nodeB = .2. + endpoint = "192.168.1.1:51820"; meshIp = "10.42.0.1"; }; b = { publicKey = testKeys.b.publicKey; - endpoint = "192.168.1.3:51820"; + endpoint = "192.168.1.2:51820"; meshIp = "10.42.0.2"; }; }; diff --git a/test/regtest-edge.nix b/test/regtest-edge.nix index edaf428..80e820e 100644 --- a/test/regtest-edge.nix +++ b/test/regtest-edge.nix @@ -49,11 +49,12 @@ let # `user:password` line the edge feeds frigate via LoadCredential. authCredentialFile = pkgs.writeText "edge-bitcoind-auth" "${rpcUser}:${rpcPassword}"; - # Mesh-like IP shared between the two VMs. The nixosTest default - # subnet is 192.168.1.0/24 with the first declared node at .2; pin - # the backend's IP via test-driver options so the edge can address it - # at a known location regardless of order. - backendIp = "192.168.1.2"; + # The nixosTest framework assigns 192.168.. + # starting at nodeNumber 1, in node-declaration order: `backend` is + # declared first so it ends up at .1, and `edge` at .2. Hardcoded + # here because the edge config needs to reference the backend's + # address before the test driver has wired up the topology. + backendIp = "192.168.1.1"; in pkgs.testers.runNixOSTest { name = "regtest-edge"; From b84c3307996e1aa4e52606b69c7a5982f6975cb5 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 13:13:27 +0200 Subject: [PATCH 6/8] fix: use correct HMAC algorithm in regtest-edge fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous HMAC was computed with `openssl ... -macopt hexkey:$salt`, which decodes the salt from hex into raw bytes and uses those bytes as the HMAC key. bitcoind's rpcauth.py uses the salt's literal UTF-8 string bytes as the key: hmac.new(salt.encode("utf-8"), password.encode("utf-8"), "SHA256") Two different keys → two different HMACs. Every auth attempt the edge sent to bitcoind was rejected with "incorrect password attempt". Recomputed with the correct algorithm. Comment now states the exact key derivation so the next person who hits this doesn't trip over the same mistake. Verifiable via: python3 -c "import hmac; print(hmac.new('2316d0a5e8ee6339ffb4d86c983bb421'.encode(), 'testpassword'.encode(), 'SHA256').hexdigest())" # → 34cc4776187170b359d40928b25deb28ea2bfc436c96fdd0db7150ec5211de85 Co-Authored-By: Claude Opus 4.7 (1M context) --- test/regtest-edge.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/regtest-edge.nix b/test/regtest-edge.nix index 80e820e..ebda28e 100644 --- a/test/regtest-edge.nix +++ b/test/regtest-edge.nix @@ -41,10 +41,12 @@ let rpcUser = "frigate-edge"; rpcPassword = "testpassword"; - # `salt$hash` computed once via HMAC-SHA256(salt, password). Same - # algorithm bitcoind/share/rpcauth/rpcauth.py implements. Committable - # — derives one-way from the plaintext. - rpcPasswordHMAC = "2316d0a5e8ee6339ffb4d86c983bb421$9b90ff10a12e7df0dee2cd86f827461d3a481f1947a0bae613e4046407ee6ced"; + # `salt$hash` form bitcoind expects. The HMAC is computed as + # HMAC-SHA256 with the salt's *literal UTF-8 bytes* as the key (not + # the hex-decoded bytes) — same algorithm bitcoind/share/rpcauth/rpcauth.py + # implements: `hmac.new(salt.encode("utf-8"), password.encode("utf-8"), "SHA256")`. + # Committable — derives one-way from the plaintext. + rpcPasswordHMAC = "2316d0a5e8ee6339ffb4d86c983bb421$34cc4776187170b359d40928b25deb28ea2bfc436c96fdd0db7150ec5211de85"; # `user:password` line the edge feeds frigate via LoadCredential. authCredentialFile = pkgs.writeText "edge-bitcoind-auth" "${rpcUser}:${rpcPassword}"; From e9a61ad6d9cd7f4e11d9ae9c91bc60ffd4ca6101 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 13:26:34 +0200 Subject: [PATCH 7/8] test: surface frigate restart-loop in regtest-edge with journal dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `wait_for_open_port` defaults to 900s. When frigate fails its first connect to bitcoind (auth error, DNS, ZMQ) systemd's Restart=on-failure keeps trying every ~13s, and with DefaultStartLimitBurst=5 / DefaultStartLimitIntervalSec=10s the burst limit never trips — the restarts are spaced just far enough apart. The unit stays in eternal auto-restart while the port never opens; the test waits 15 minutes and then fails with a useless "port never opened" message. Replace the bare `wait_for_open_port(50001)` with a 60s polling loop that, on timeout, dumps the last 50 lines of frigate's journal. The 60s bound covers ~4–5 restart cycles — plenty for a legitimately slow backend boot, and short enough to surface a real configuration bug quickly. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/regtest-edge.nix | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/test/regtest-edge.nix b/test/regtest-edge.nix index ebda28e..94afe1c 100644 --- a/test/regtest-edge.nix +++ b/test/regtest-edge.nix @@ -188,9 +188,28 @@ pkgs.testers.runNixOSTest { # Frigate-edge should authenticate against bitcoind (USERPASS, # plaintext fed via LoadCredential), subscribe to remote ZMQ, and # accept Electrum traffic on both its plaintext and TLS listeners. + # + # `wait_for_unit` accepts the unit in the brief `activating` state + # of a restart cycle, so it's not a strong "frigate is up" signal. + # The port-open check is the real gate. Bound it tightly enough + # that a stuck restart loop surfaces fast, and dump the unit + # journal on failure so the actual error (auth, DNS, ZMQ) is + # visible in CI output instead of just "port never opened". edge.wait_for_unit("frigate.service") - edge.wait_for_open_port(50001) - edge.wait_for_open_port(50002) + + import time + deadline = time.time() + 60 + while time.time() < deadline: + if edge.execute("ss -tln | grep -q ':50001 '")[0] == 0: + break + time.sleep(2) + else: + journal, _ = edge.execute("journalctl -u frigate -n 50 --no-pager") + raise Exception( + f"frigate-edge did not bind port 50001 within 60s. " + f"Last 50 journal lines:\n{journal}" + ) + edge.wait_for_open_port(50002, timeout=10) import time deadline = time.time() + 120 From eee125714fb1a145f0566227061b1a9abe6d2ca8 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 19 May 2026 13:35:51 +0200 Subject: [PATCH 8/8] test: install netcat-openbsd on edge so the probe's nc -q works The probe pipes Electrum JSON-RPC into `nc -q 3 127.0.0.1 50001`. The `-q` flag (wait N seconds after stdin EOF before closing) is a netcat-openbsd extension; NixOS's default nc supports `-z` but not `-q`, so the probe silently emitted nothing and the 120s loop timed out with empty responses on every iteration. regtest-preset.nix avoids this by adding `pkgs.netcat-openbsd` to `environment.systemPackages`. Mirror that here on the edge node. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/regtest-edge.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/regtest-edge.nix b/test/regtest-edge.nix index 94afe1c..101545d 100644 --- a/test/regtest-edge.nix +++ b/test/regtest-edge.nix @@ -147,6 +147,12 @@ pkgs.testers.runNixOSTest { # what regtest-preset does on the backend. services.frigate.computeBackend = lib.mkForce "CPU"; + # The probe below pipes JSON-RPC into `nc -q 3` to bound how long + # nc waits after stdin EOF. `-q` is a netcat-openbsd extension; + # NixOS's default nc supports `-z` but not `-q`, so without this + # package the probe silently emits nothing and the loop times out. + environment.systemPackages = [ pkgs.netcat-openbsd ]; + virtualisation.cores = 2; virtualisation.memorySize = 2048; };