From d56e6ff8799ef959c6c42fac06ffc7334b64826a Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:23:10 +0200 Subject: [PATCH 01/99] fix(network): stop tagging unverified peer-announce peers into peerstore --- backend/src/protocol/peer-announce.ts | 33 ++++++++------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index 1d98348d9..7faf31a85 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -1,7 +1,6 @@ import { trace } from '../logger.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; -import { peerIdFromString as peerIDFromString } from '@libp2p/peer-id'; import { LISH_TOPIC_PREFIX } from './constants.ts'; import { type Libp2p } from 'libp2p'; import { type BootstrapPeerOrigin } from '@shared'; @@ -152,30 +151,16 @@ export class PeerAnnounceManager { // Pass networkID so per-peer outcomes from gossiped entries surface in the // UI under the network through which they arrived. Identity-mismatch // outcomes inside addBootstrapPeers also trigger purgeStalePeer. + // + // Deliberately NO peerStore.merge / keep-alive tagging here: an announce is + // an unverified claim. Writing tags for every mentioned peerID would (a) let + // any topic subscriber inject arbitrary peerIDs that we then persist and + // re-dial forever, and (b) refresh peerStore maxPeerAge for long-dead peers + // on every cycle, so they never expire. Keep-alive tagging happens only + // after a dial actually succeeds (addBootstrapPeers, peer:connect, re-dial + // maintenance) — a dead peer mentioned by gossip is still dialed below, it + // just no longer leaves a permanent peerStore footprint when unreachable. await this.deps.addBootstrapPeers(filtered, networkID, 'discovered'); - // Stamp `keep-alive-fleet` on every peer the announce mentioned. libp2p - // ReconnectQueue only acts on peers carrying a tag whose key starts with - // `keep-alive`; without this tag, fleet-discovered peers that drop are - // not re-dialed automatically. addBootstrapPeers() above tags via KEEP_ALIVE - // only for peer IDs it successfully extracts from multiaddrs — this adds - // the same treatment for every known peer, driving mesh maintenance. - const node = this.deps.getNode(); - if (node) { - for (const ma of filtered) { - try { - const mapath = Multiaddr(ma); - const pidComp = mapath.getComponents().find(c => c.code === 421); - const pid = pidComp?.value; - if (!pid) continue; - if (pid === node.peerId.toString()) continue; - await node.peerStore.merge(peerIDFromString(pid), { - tags: { 'keep-alive-fleet': { value: 50 } }, - }); - } catch { - /* invalid multiaddr — skip */ - } - } - } } private async scheduleNext(): Promise { From 177c6210f18efad30ffec228a53522872783f977 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:23:31 +0200 Subject: [PATCH 02/99] fix(network): purge stale peers from autodial list and gossipsub direct set --- backend/src/protocol/network.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index a8bcabace..63d1b0885 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1001,6 +1001,20 @@ export class Network { async purgeStalePeer(peerID: string, reason: string): Promise { if (!this.node) return; this.bootstrapPeerIDs.delete(peerID); + // Drop the peer's addrs from the autodial list too — this array is otherwise + // push-only, so the zero-connection recovery loop would keep dialing addrs + // of an identity we just proved dead, and the array would grow until stop(). + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => { + try { + return ma.getComponents().find((c: any) => c.code === 421)?.value !== peerID; + } catch { + return true; + } + }); + // Remove from the gossipsub never-PRUNE direct set, or gossipsub keeps + // attempting a direct stream to the dead peer every directConnectTicks. + const gossipsub: any = this.pubsub; + if (gossipsub?.direct && typeof gossipsub.direct.delete === 'function') gossipsub.direct.delete(peerID); try { const pid = peerIDFromString(peerID); // Drop existing connections so libp2p considers the entry fully gone. From 82780bed1019e7ba677ea0489548bfe2a22a3e8a Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:28:03 +0200 Subject: [PATCH 03/99] fix(network): evict peers unreachable across repeated redials --- backend/src/protocol/bootstrap-status.ts | 21 ++++++ backend/src/protocol/network.ts | 67 ++++++++++++++++++- .../unit/protocol/bootstrap-status.test.ts | 48 +++++++++++++ 3 files changed, 134 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index e0356da0a..84b9c0c8b 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -74,6 +74,27 @@ export class BootstrapStatusTracker { this.onStatusChange?.(networkID, snap); } + /** + * Drop every discovered-origin entry recorded for the given peer identity, in + * every network. Used when a peer is evicted as unreachable — its gossip-learned + * rows are pure noise at that point. Configured rows are kept: they are user + * data and must stay visible (red) so the user can fix or remove them. + */ + deleteDiscoveredByPeerID(peerID: string): void { + for (const [networkID, peers] of [...this.stats]) { + let changed = false; + for (const [addr, p] of [...peers]) { + if (p.origin !== 'discovered') continue; + if (p.expectedPeerID !== peerID && p.actualPeerID !== peerID) continue; + peers.delete(addr); + changed = true; + } + if (!changed) continue; + if (peers.size === 0) this.stats.delete(networkID); + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + } + /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ pruneEntries(networkID: string, keepMultiaddrs: string[]): void { const peers = this.stats.get(networkID); diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 63d1b0885..48f5ae219 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -81,6 +81,22 @@ const WANT_RESPONSE_COOLDOWN_MS = 60_000; const WANT_RESPONSE_CLEANUP_INTERVAL_MS = 5 * 60_000; /** Search query dedup window — same `searchID` arriving via mesh within this period is ignored. */ const SEARCH_DEDUP_TTL_MS = 5 * 60_000; +/** + * Consecutive re-dial failures after which a peer is treated as gone and evicted + * (peerStore + bootstrap sets + its discovered status rows). Combined with + * REDIAL_EVICT_MIN_MS so a burst of quick failures right after our own restart + * or a network partition cannot mass-purge peers that are merely slow to return. + */ +const REDIAL_EVICT_FAILS = 6; +/** Minimum continuous unreachability (since the first recorded failure) before eviction. */ +const REDIAL_EVICT_MIN_MS = 30 * 60_000; +/** + * How long an evicted-as-unreachable peer stays quarantined in addBootstrapPeers. + * Gossip from nodes that still remember the dead peer keeps mentioning it; without + * this window every mention would re-create its status row and burn a dial. Once + * the window lapses a single probe is allowed again (self-heals on peer return). + */ +const UNREACHABLE_QUARANTINE_MS = 60 * 60_000; /** * Maximum size (bytes) of an incoming pubsub payload we are willing to decode. * Our own control messages ride pubsub (WANT — tiny JSON), but older/foreign peers @@ -187,7 +203,16 @@ export class Network { * capped at 10 min), so a persistently-unreachable peer does not saturate the * re-dial pool every 30s. Successful dial clears the entry. */ - private readonly redialBackoff = new Map(); + private readonly redialBackoff = new Map(); + /** peerID → eviction time. Blocks re-adding a just-evicted unreachable peer from gossip for UNREACHABLE_QUARANTINE_MS. */ + private readonly unreachableQuarantine = new Map(); + /** + * Peer IDs that appear in at least one network's CONFIGURED bootstrap list. + * These are user data — the unreachable-eviction path must never purge them, + * or a bootstrap hub that is down for half an hour would lose its peerStore + * entry and its addrs in bootstrapMultiaddrs until the next restart. + */ + private readonly configuredPeerIDs = new Set(); // Tracked libp2p/pubsub event listeners for clean removal in stop(). // Each entry captures the exact handler reference so removeEventListener can unhook it. @@ -508,6 +533,7 @@ export class Network { this.addListener(this.node!, 'peer:connect', async (evt: any) => { try { const peerID = evt.detail.toString(); + this.unreachableQuarantine.delete(peerID); const connections = this.node!.getConnections(evt.detail); const connTypes = connections.map(c => { const isRelay = Circuit.matches(c.remoteAddr); @@ -697,6 +723,7 @@ export class Network { const pid = peer.id.toString(); if (connectedSet.has(pid)) { this.redialBackoff.delete(pid); // clear on observed connection + this.unreachableQuarantine.delete(pid); continue; } const bo = this.redialBackoff.get(pid); @@ -758,8 +785,22 @@ export class Network { // Exponential backoff: 30s × 2^failCount, capped at 10 min. const nextFailCount = c.failCount + 1; const delayMs = Math.min(30_000 * 2 ** c.failCount, 600_000); - this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount }); + const firstFailure = this.redialBackoff.get(c.pid)?.firstFailure ?? Date.now(); + this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount, firstFailure }); console.debug(` ✗ Re-dial peer=${c.pid} failed: ${err.message ?? err} (tried: ${c.addrSummary}, next in ${Math.round(delayMs / 1000)}s)`); + // Enough consecutive failures over enough time ⇒ the peer is gone, not + // flaky. The dial above went by peer ID, so libp2p tried EVERY known + // address — one broken addr among working ones cannot trip this. Evict + // everywhere (peerStore, bootstrap sets, discovered status rows) and + // quarantine the ID so gossip mentions don't immediately re-add it. + // Configured bootstrap peers are exempt — user data, they must survive + // any outage and keep their red status row instead. + if (nextFailCount >= REDIAL_EVICT_FAILS && Date.now() - firstFailure >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(c.pid)) { + this.unreachableQuarantine.set(c.pid, Date.now()); + this.redialBackoff.delete(c.pid); + this.bootstrapTracker.deleteDiscoveredByPeerID(c.pid); + await this.purgeStalePeer(c.pid, `unreachable after ${nextFailCount} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`); + } } } }; @@ -771,6 +812,10 @@ export class Network { // Prune backoff entries for peers that are no longer in peerStore to prevent unbounded growth. const storeSet = new Set(allPeers.map(p => p.id.toString())); for (const pid of this.redialBackoff.keys()) if (!storeSet.has(pid)) this.redialBackoff.delete(pid); + // Quarantine entries for peers gossip never mentions again would leak — drop + // them once they are far past the window (re-entry from gossip self-cleans). + const quarantineCutoff = now - 2 * UNREACHABLE_QUARANTINE_MS; + for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } private async runZeroConnectionRecovery(connectedPeers: any[]): Promise { @@ -918,6 +963,21 @@ export class Network { continue; } const peerID = ma.getComponents().find(c => c.code === 421)?.value ?? null; + if (peerID && origin === 'configured') this.configuredPeerIDs.add(peerID); + // Skip peers recently evicted as unreachable — nodes that still remember + // them keep gossiping their addrs, and without this window every mention + // would re-create the status row and burn a dial. Configured entries are + // exempt: the user asked for them explicitly. + if (peerID && origin === 'discovered') { + const quarantinedAt = this.unreachableQuarantine.get(peerID); + if (quarantinedAt !== undefined) { + if (Date.now() - quarantinedAt < UNREACHABLE_QUARANTINE_MS) { + trace(`[NET] addBootstrapPeers skip quarantined: ${peerID.slice(0, 16)}`); + continue; + } + this.unreachableQuarantine.delete(peerID); + } + } const alreadyKnown = !!peerID && this.bootstrapPeerIDs.has(peerID); if (peerID && !alreadyKnown) { this.bootstrapPeerIDs.add(peerID); @@ -1015,6 +1075,7 @@ export class Network { // attempting a direct stream to the dead peer every directConnectTicks. const gossipsub: any = this.pubsub; if (gossipsub?.direct && typeof gossipsub.direct.delete === 'function') gossipsub.direct.delete(peerID); + this.redialBackoff.delete(peerID); try { const pid = peerIDFromString(peerID); // Drop existing connections so libp2p considers the entry fully gone. @@ -1430,6 +1491,8 @@ export class Network { this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); + this.unreachableQuarantine.clear(); + this.configuredPeerIDs.clear(); this.pxIngressLogKeys.clear(); if (this.node) { await this.node.stop(); diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 4ad293c83..8a8ce4cd6 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { classifyBootstrapError, extractActualPeerID } from '../../../src/protocol/network.ts'; +import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; // Deterministic unit tests for the bootstrap-peer dial classification — the pure // logic that decides whether a failed bootstrap dial is an identity-mismatch (stale @@ -57,3 +58,50 @@ describe('extractActualPeerID', () => { expect(extractActualPeerID('does not match expected remote identity key only')).toBe(null); }); }); + +describe('BootstrapStatusTracker.deleteDiscoveredByPeerID', () => { + const NET_A = 'netAAAA'; + const NET_B = 'netBBBB'; + const DEAD_ID = '12D3KooWDeadDeadDeadDeadDeadDeadDeadDeadDeadDeadDD'; + const LIVE_ID = '12D3KooWLiveLiveLiveLiveLiveLiveLiveLiveLiveLiveLL'; + const DEAD_ADDR_1 = `/ip4/192.0.2.10/tcp/9090/p2p/${DEAD_ID}`; + const DEAD_ADDR_2 = `/ip4/192.0.2.11/tcp/9090/p2p/${DEAD_ID}`; + const LIVE_ADDR = `/ip4/192.0.2.20/tcp/9090/p2p/${LIVE_ID}`; + + it('removes discovered rows for the peer across all networks, keeps other peers', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, DEAD_ADDR_1, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + tracker.recordOutcome(NET_A, LIVE_ADDR, LIVE_ID, 'connected', null, null, 'discovered'); + tracker.recordOutcome(NET_B, DEAD_ADDR_2, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)?.peers.map(p => p.multiaddr)).toEqual([LIVE_ADDR]); + expect(tracker.getStatus(NET_B)).toBe(null); // network map emptied entirely + }); + + it('keeps configured rows for the same peer identity', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, DEAD_ADDR_1, DEAD_ID, 'timeout', 'The operation timed out', null, 'configured'); + tracker.recordOutcome(NET_A, DEAD_ADDR_2, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)?.peers.map(p => p.multiaddr)).toEqual([DEAD_ADDR_1]); + }); + + it('matches rows by actualPeerID as well and fires onStatusChange per changed network', () => { + const tracker = new BootstrapStatusTracker(); + const events: string[] = []; + tracker.setOnChange(networkID => events.push(networkID)); + // Row whose expectedPeerID is null but whose dial revealed the dead identity. + tracker.recordOutcome(NET_A, '/ip4/192.0.2.30/tcp/9090', null, 'identity-mismatch', 'mismatch', DEAD_ID, 'discovered'); + tracker.recordOutcome(NET_B, LIVE_ADDR, LIVE_ID, 'connected', null, null, 'discovered'); + events.length = 0; + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)).toBe(null); + expect(events).toEqual([NET_A]); // untouched NET_B emits nothing + }); +}); From b3f49a22d95873f9f277560fe2d4b5b1da4bac40 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:29:18 +0200 Subject: [PATCH 04/99] fix(settings): expire stale discovered peers from bootstrap status --- backend/src/protocol/bootstrap-status.ts | 25 +++++++++++ backend/src/protocol/network.ts | 8 ++++ .../unit/protocol/bootstrap-status.test.ts | 42 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 84b9c0c8b..5dee81659 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -95,6 +95,31 @@ export class BootstrapStatusTracker { } } + /** + * Drop discovered-origin entries that have gone stale: no status refresh within + * `ttlMs` AND no current connection to the peer. Dead peers stop being mentioned + * by gossip, so their rows stop refreshing and expire here — including rows + * frozen at 'connected' for a peer that silently died. Configured entries are + * exempt (user data). `now` is injectable for tests. + */ + sweepStale(ttlMs: number, isConnected: (peerID: string) => boolean, now: number = Date.now()): void { + for (const [networkID, peers] of [...this.stats]) { + let changed = false; + for (const [addr, p] of [...peers]) { + if (p.origin !== 'discovered') continue; + const pid = p.expectedPeerID ?? p.actualPeerID; + if (pid && isConnected(pid)) continue; + const updated = Date.parse(p.updatedAt); + if (Number.isFinite(updated) && now - updated < ttlMs) continue; + peers.delete(addr); + changed = true; + } + if (!changed) continue; + if (peers.size === 0) this.stats.delete(networkID); + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + } + /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ pruneEntries(networkID: string, keepMultiaddrs: string[]): void { const peers = this.stats.get(networkID); diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 48f5ae219..b376dddea 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -90,6 +90,12 @@ const SEARCH_DEDUP_TTL_MS = 5 * 60_000; const REDIAL_EVICT_FAILS = 6; /** Minimum continuous unreachability (since the first recorded failure) before eviction. */ const REDIAL_EVICT_MIN_MS = 30 * 60_000; +/** + * Discovered bootstrap-status rows older than this (and without a live connection) + * are dropped from the UI. Live peers keep refreshing their rows via gossip intake; + * dead ones stop being mentioned, freeze, and expire here. + */ +const BOOTSTRAP_STATUS_STALE_MS = 30 * 60_000; /** * How long an evicted-as-unreachable peer stays quarantined in addBootstrapPeers. * Gossip from nodes that still remember the dead peer keeps mentioning it; without @@ -699,6 +705,8 @@ export class Network { await this.runRedialMaintenance(connectedPeers, allPeers); await this.runZeroConnectionRecovery(connectedPeers); await this.maybePromotePeers(); + const connectedIDs = new Set(connectedPeers.map((p: any) => p.toString())); + this.bootstrapTracker.sweepStale(BOOTSTRAP_STATUS_STALE_MS, pid => connectedIDs.has(pid)); } catch (err: any) { trace(`[NET] statusInterval error: ${err?.message ?? err}`); } diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 8a8ce4cd6..98432a331 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -105,3 +105,45 @@ describe('BootstrapStatusTracker.deleteDiscoveredByPeerID', () => { expect(events).toEqual([NET_A]); // untouched NET_B emits nothing }); }); + +describe('BootstrapStatusTracker.sweepStale', () => { + const NET = 'netAAAA'; + const TTL = 30 * 60_000; + const DEAD_ID = '12D3KooWDeadDeadDeadDeadDeadDeadDeadDeadDeadDeadDD'; + const LIVE_ID = '12D3KooWLiveLiveLiveLiveLiveLiveLiveLiveLiveLiveLL'; + const DEAD_ADDR = `/ip4/192.0.2.10/tcp/9090/p2p/${DEAD_ID}`; + const LIVE_ADDR = `/ip4/192.0.2.20/tcp/9090/p2p/${LIVE_ID}`; + const CONF_ADDR = `/ip4/192.0.2.30/tcp/9090/p2p/${DEAD_ID}`; + + it('drops stale discovered rows, keeps fresh, connected and configured ones', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + tracker.recordOutcome(NET, LIVE_ADDR, LIVE_ID, 'connected', null, null, 'discovered'); + tracker.recordOutcome(NET, CONF_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'configured'); + const past = Date.now() + TTL + 60_000; // both rows are then older than TTL + + tracker.sweepStale(TTL, pid => pid === LIVE_ID, past); + + const addrs = tracker.getStatus(NET)?.peers.map(p => p.multiaddr).sort(); + // DEAD discovered row expired; LIVE row survives via connection; configured row untouchable. + expect(addrs).toEqual([CONF_ADDR, LIVE_ADDR].sort()); + }); + + it('drops a row frozen at connected once the peer has no live connection', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, () => false, Date.now() + TTL + 60_000); + + expect(tracker.getStatus(NET)).toBe(null); + }); + + it('keeps rows within the TTL even without a connection', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.sweepStale(TTL, () => false); // real clock — row was written moments ago + + expect(tracker.getStatus(NET)?.peers.length).toBe(1); + }); +}); From b778c154fd80c50f28b012e88e528e185880afd5 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:30:54 +0200 Subject: [PATCH 05/99] fix(network): promote only connected peers to bootstrap set --- backend/src/protocol/network.ts | 38 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index b376dddea..abaeb0f95 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -862,10 +862,8 @@ export class Network { private async maybePromotePeers(): Promise { // Every 5th status tick (~150 s at 30 s status cadence) promote every - // peerStore entry back to bootstrap priority. Re-stamps KEEP_ALIVE tags - // and feeds libp2p a concrete multiaddr list to re-dial against, catching - // peers whose original dial cached a stale (unreachable) address — these - // would otherwise sit idle until they reappeared via identify/PX/announce. + // CONNECTED peer back to bootstrap priority (KEEP_ALIVE re-stamp + gossipsub + // direct set). Disconnected peers are handled by runRedialMaintenance. this.statusTickCount++; if (this.statusTickCount % 5 === 0) { try { @@ -877,19 +875,24 @@ export class Network { } /** - * Promote every known peer (from libp2p peerStore) back to bootstrap priority so - * KEEP_ALIVE tagging and direct-dial re-runs cover peers the ordinary re-dial loop - * skipped because their cached multiaddrs looked like loopback/private-IP garbage. - * Runs every ~45 s from the status tick. + * Promote every CONNECTED peer back to bootstrap priority: KEEP_ALIVE tagging, + * bootstrap dedup-set membership, and gossipsub direct-set fast reconnect. + * Disconnected peers are deliberately excluded — runRedialMaintenance already + * dials each of them every tick with exponential backoff and eviction, whereas + * promotion dials have no backoff, so including them meant a burst of dials to + * dead peers every promotion cycle and their permanent growth in the direct set. + * Runs every ~150 s from the status tick. */ private async promoteKnownPeersToBootstrap(): Promise { if (!this.node) return; const allPeers = await this.node.peerStore.all(); const myID = this.node.peerId.toString(); + const connectedIDs = new Set(this.node.getPeers().map((p: any) => p.toString())); const maStrings: string[] = []; for (const peer of allPeers) { const pid = peer.id.toString(); if (pid === myID) continue; + if (!connectedIDs.has(pid)) continue; if (this.bootstrapPeerIDs.has(pid)) continue; if (peer.addresses.length === 0) continue; const addr = peer.addresses[0]!; @@ -898,25 +901,28 @@ export class Network { const maStr = base.includes('/p2p/') ? base : `${base}/p2p/${pid}`; maStrings.push(maStr); } - if (maStrings.length === 0) return; - trace(`[NET] periodic autodial: promoting ${maStrings.length} peer(s) to bootstrap`); - await this.addBootstrapPeers(maStrings); - // Also insert every known peer into the gossipsub `direct` Set at runtime. - // Direct peers are never PRUNED by D/Dhi and have their own fast reconnect - // cadence (directConnectTicks × heartbeatInterval). KEEP_ALIVE handles the - // TCP layer; gossipsub.direct handles the gossipsub-stream layer. + if (maStrings.length > 0) { + trace(`[NET] periodic autodial: promoting ${maStrings.length} connected peer(s) to bootstrap`); + await this.addBootstrapPeers(maStrings); + } + // Also insert every connected peer into the gossipsub `direct` Set at runtime. + // Direct peers have their own fast reconnect cadence (directConnectTicks × + // heartbeatInterval). KEEP_ALIVE handles the TCP layer; gossipsub.direct + // handles the gossipsub-stream layer. Evicted peers are removed from the + // set in purgeStalePeer, so it no longer grows monotonically. const gossipsub: any = this.pubsub; if (gossipsub?.direct && typeof gossipsub.direct.add === 'function') { let added = 0; for (const peer of allPeers) { const pid = peer.id.toString(); if (pid === myID) continue; + if (!connectedIDs.has(pid)) continue; if (!gossipsub.direct.has(pid)) { gossipsub.direct.add(pid); added++; } } - if (added > 0) trace(`[NET] gossipsub direct: added ${added} known peer(s) to never-PRUNE set`); + if (added > 0) trace(`[NET] gossipsub direct: added ${added} connected peer(s) to fast-reconnect set`); } } From ebf685746f4c9e632979fbd6c895ea4bd6746bf2 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 08:32:26 +0200 Subject: [PATCH 06/99] style: prettier reformatting (no logic change) --- backend/tests/unit/protocol/bootstrap-status.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 98432a331..ade701675 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -124,7 +124,10 @@ describe('BootstrapStatusTracker.sweepStale', () => { tracker.sweepStale(TTL, pid => pid === LIVE_ID, past); - const addrs = tracker.getStatus(NET)?.peers.map(p => p.multiaddr).sort(); + const addrs = tracker + .getStatus(NET) + ?.peers.map(p => p.multiaddr) + .sort(); // DEAD discovered row expired; LIVE row survives via connection; configured row untouchable. expect(addrs).toEqual([CONF_ADDR, LIVE_ADDR].sort()); }); From 2681c9f9ecaf90cdf428198aa3a67426d4e76b25 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 14:53:13 +0200 Subject: [PATCH 07/99] fix(network): extract destination peer ID from circuit multiaddrs --- backend/src/protocol/network.ts | 36 +++++++++++++------ .../unit/protocol/bootstrap-status.test.ts | 26 +++++++++++++- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index abaeb0f95..95acd7e36 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -897,8 +897,10 @@ export class Network { if (peer.addresses.length === 0) continue; const addr = peer.addresses[0]!; const base = addr.multiaddr.toString(); - // Ensure /p2p/ suffix — addBootstrapPeers extracts peer ID via multiaddr component 421. - const maStr = base.includes('/p2p/') ? base : `${base}/p2p/${pid}`; + // Ensure the address terminates in THIS peer's /p2p/ — a bare address + // gets the suffix appended, and so does a relay address whose only /p2p/ + // component is the relay's own identity. + const maStr = extractDestinationPeerID(addr.multiaddr) === pid ? base : `${base}/p2p/${pid}`; maStrings.push(maStr); } if (maStrings.length > 0) { @@ -976,7 +978,7 @@ export class Network { trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); continue; } - const peerID = ma.getComponents().find(c => c.code === 421)?.value ?? null; + const peerID = extractDestinationPeerID(ma); if (peerID && origin === 'configured') this.configuredPeerIDs.add(peerID); // Skip peers recently evicted as unreachable — nodes that still remember // them keep gossiping their addrs, and without this window every mention @@ -1078,13 +1080,7 @@ export class Network { // Drop the peer's addrs from the autodial list too — this array is otherwise // push-only, so the zero-connection recovery loop would keep dialing addrs // of an identity we just proved dead, and the array would grow until stop(). - this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => { - try { - return ma.getComponents().find((c: any) => c.code === 421)?.value !== peerID; - } catch { - return true; - } - }); + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => extractDestinationPeerID(ma) !== peerID); // Remove from the gossipsub never-PRUNE direct set, or gossipsub keeps // attempting a direct stream to the dead peer every directConnectTicks. const gossipsub: any = this.pubsub; @@ -1553,6 +1549,26 @@ export class Network { * - `error`: every other reason (invalid multiaddr, connection refused, protocol * negotiation failure, etc). */ +/** + * Extract the DESTINATION peer ID from a multiaddr. A circuit-relay address has + * the shape `/.../p2p//p2p-circuit/p2p/` — taking the FIRST + * /p2p/ component would return the relay's identity, so eviction and configured + * protection would target the wrong peer. The last /p2p/ component is always + * the dial target. Returns null when the multiaddr carries no peer ID at all. + */ +export function extractDestinationPeerID(ma: any): string | null { + try { + const components: Array<{ code: number; value?: string }> = ma?.getComponents?.() ?? []; + for (let i = components.length - 1; i >= 0; i--) { + const c = components[i]!; + if (c.code === 421 && typeof c.value === 'string') return c.value; + } + } catch { + /* unparseable multiaddr — no ID */ + } + return null; +} + export function classifyBootstrapError(message: string): BootstrapPeerDialStatus { if (!message) return 'error'; if (message.includes('does not match expected remote identity key')) return 'identity-mismatch'; diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index ade701675..cbda29f9a 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'bun:test'; -import { classifyBootstrapError, extractActualPeerID } from '../../../src/protocol/network.ts'; +import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; +import { classifyBootstrapError, extractActualPeerID, extractDestinationPeerID } from '../../../src/protocol/network.ts'; import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; // Deterministic unit tests for the bootstrap-peer dial classification — the pure @@ -59,6 +60,29 @@ describe('extractActualPeerID', () => { }); }); +describe('extractDestinationPeerID', () => { + // Real base58 ed25519 peer IDs are required — the multiaddr parser validates them. + const RELAY_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const DST_ID = '12D3KooWAnfqA6Wap96ixVfxhHeGUDMriBG4Nncp5tqu8q71EVv2'; + + it('returns the terminal peer ID of a plain address', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${DST_ID}`))).toBe(DST_ID); + }); + + it('returns the DESTINATION (not the relay) for a circuit address', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${RELAY_ID}/p2p-circuit/p2p/${DST_ID}`))).toBe(DST_ID); + }); + + it('returns the relay ID when a circuit address has no destination component', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${RELAY_ID}/p2p-circuit`))).toBe(RELAY_ID); + }); + + it('returns null for an address without any peer ID and for garbage input', () => { + expect(extractDestinationPeerID(Multiaddr('/ip4/192.0.2.1/tcp/9090'))).toBe(null); + expect(extractDestinationPeerID(null)).toBe(null); + }); +}); + describe('BootstrapStatusTracker.deleteDiscoveredByPeerID', () => { const NET_A = 'netAAAA'; const NET_B = 'netBBBB'; From febfc2e5e6cc5f2c51a546534336049a4dec1ee1 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 14:53:51 +0200 Subject: [PATCH 08/99] fix(network): reset redial failure history on live connections --- backend/src/protocol/network.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 95acd7e36..ab0267ac0 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -217,6 +217,12 @@ export class Network { * These are user data — the unreachable-eviction path must never purge them, * or a bootstrap hub that is down for half an hour would lose its peerStore * entry and its addrs in bootstrapMultiaddrs until the next restart. + * + * Grow-only by design: entries are not removed when a bootstrap row is + * deleted or its network disabled, so a formerly-configured peer stays + * eviction-exempt until restart. That errs on the safe side (a peer is + * merely redialed longer than necessary); per-network refcounting would be + * required to shrink it correctly and is not worth the bookkeeping. */ private readonly configuredPeerIDs = new Set(); @@ -540,6 +546,12 @@ export class Network { try { const peerID = evt.detail.toString(); this.unreachableQuarantine.delete(peerID); + // Any verified connection resets the failure history — without this, a + // flappy NAT/relay peer that connects and drops BETWEEN status ticks + // keeps accumulating failCount across its live episodes and eventually + // gets evicted as "unreachable for 30 minutes" despite never being + // gone that long. + this.redialBackoff.delete(peerID); const connections = this.node!.getConnections(evt.detail); const connTypes = connections.map(c => { const isRelay = Circuit.matches(c.remoteAddr); @@ -804,6 +816,14 @@ export class Network { // Configured bootstrap peers are exempt — user data, they must survive // any outage and keep their red status row instead. if (nextFailCount >= REDIAL_EVICT_FAILS && Date.now() - firstFailure >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(c.pid)) { + // Last-moment liveness check: the peer may have connected (inbound + // dial, another async path) while this worker was failing on stale + // state. purgeStalePeer closes connections, so evicting here would + // cut a LIVE peer — verify emptiness right before acting. + if (this.node && this.node.getConnections(c.peer.id).length > 0) { + this.redialBackoff.delete(c.pid); + continue; + } this.unreachableQuarantine.set(c.pid, Date.now()); this.redialBackoff.delete(c.pid); this.bootstrapTracker.deleteDiscoveredByPeerID(c.pid); From f8be4730003321555a4fa0a2e5cc26b0113d1599 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 14:54:33 +0200 Subject: [PATCH 09/99] fix(network): serialize status ticks and use fresh sweep snapshot --- backend/src/protocol/network.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index ab0267ac0..ebff81e35 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -128,6 +128,8 @@ export class Network { private statusInterval: NodeJS.Timeout | null = null; /** Monotonic counter for status-interval ticks. Used by the periodic autodial promotion. */ private statusTickCount = 0; + /** Guards against overlapping status ticks — see setupStatusInterval. */ + private statusTickInFlight = false; /** * Per-(peer,lish) timestamp of the last `have` response we sent. * Used to rate-limit responses to repeated `want` queries from the same peer for the same LISH: @@ -707,6 +709,12 @@ export class Network { private setupStatusInterval(): void { this.statusInterval = setInterval(async () => { + // Serialize ticks: with many unreachable peers the re-dial phase (5 s + // timeout × candidates ÷ concurrency) can exceed the 30 s cadence. Two + // interleaved ticks would race on redialBackoff — one tick could evict + // (and close connections of) a peer another tick just reconnected. + if (this.statusTickInFlight) return; + this.statusTickInFlight = true; try { const connectedPeers = this.node!.getPeers(); const allPeers = await this.node!.peerStore.all(); @@ -717,10 +725,15 @@ export class Network { await this.runRedialMaintenance(connectedPeers, allPeers); await this.runZeroConnectionRecovery(connectedPeers); await this.maybePromotePeers(); - const connectedIDs = new Set(connectedPeers.map((p: any) => p.toString())); + // Fresh connection snapshot for the sweep — the tick-start snapshot is + // stale by now: a discovered peer that reconnected during the re-dial + // phase above must not have its status row swept as "not connected". + const connectedIDs = new Set(this.node!.getPeers().map((p: any) => p.toString())); this.bootstrapTracker.sweepStale(BOOTSTRAP_STATUS_STALE_MS, pid => connectedIDs.has(pid)); } catch (err: any) { trace(`[NET] statusInterval error: ${err?.message ?? err}`); + } finally { + this.statusTickInFlight = false; } }, 30000); // Status interval 30 s. promoteKnownPeersToBootstrap + gossipsub.direct From be3622f3659ff08a35b5617bf0bbdfec488d4a62 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 14:55:25 +0200 Subject: [PATCH 10/99] fix(network): evict peers with no reachable addresses after grace --- backend/src/protocol/network.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index ebff81e35..ad5d5981e 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -214,6 +214,13 @@ export class Network { private readonly redialBackoff = new Map(); /** peerID → eviction time. Blocks re-adding a just-evicted unreachable peer from gossip for UNREACHABLE_QUARANTINE_MS. */ private readonly unreachableQuarantine = new Map(); + /** + * peerID → time we first saw the peer disconnected with ZERO reachable + * addresses. Such peers never enter the re-dial path (nothing to dial), so + * the failure counter cannot evict them — without this they would sit in + * peerStore/bootstrap sets until maxPeerAge while every tick re-scans them. + */ + private readonly noReachableSince = new Map(); /** * Peer IDs that appear in at least one network's CONFIGURED bootstrap list. * These are user data — the unreachable-eviction path must never purge them, @@ -554,6 +561,7 @@ export class Network { // gets evicted as "unreachable for 30 minutes" despite never being // gone that long. this.redialBackoff.delete(peerID); + this.noReachableSince.delete(peerID); const connections = this.node!.getConnections(evt.detail); const connTypes = connections.map(c => { const isRelay = Circuit.matches(c.remoteAddr); @@ -757,6 +765,7 @@ export class Network { if (connectedSet.has(pid)) { this.redialBackoff.delete(pid); // clear on observed connection this.unreachableQuarantine.delete(pid); + this.noReachableSince.delete(pid); continue; } const bo = this.redialBackoff.get(pid); @@ -777,8 +786,22 @@ export class Network { } if (reachable.length === 0) { skippedNoReachable++; + // No dialable address ⇒ the failure counter can never fire for this + // peer. Track how long it has been in this state; a disconnected peer + // with zero reachable addrs for the whole eviction window is as gone + // as one that failed every dial. Same exemptions as the dial path. + const since = this.noReachableSince.get(pid) ?? now; + if (!this.noReachableSince.has(pid)) this.noReachableSince.set(pid, now); + if (now - since >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(pid)) { + this.noReachableSince.delete(pid); + this.unreachableQuarantine.set(pid, now); + this.redialBackoff.delete(pid); + this.bootstrapTracker.deleteDiscoveredByPeerID(pid); + await this.purgeStalePeer(pid, `no reachable addresses for ${Math.round((now - since) / 60_000)} min`); + } continue; } + this.noReachableSince.delete(pid); // addresses came back — reset the clock candidates.push({ peer, pid, addrSummary: reachable.join(' | '), failCount: bo?.failCount ?? 0 }); } // Parallel dial with concurrency=10 via rolling promise pool; caps worst-case @@ -853,6 +876,7 @@ export class Network { // Prune backoff entries for peers that are no longer in peerStore to prevent unbounded growth. const storeSet = new Set(allPeers.map(p => p.id.toString())); for (const pid of this.redialBackoff.keys()) if (!storeSet.has(pid)) this.redialBackoff.delete(pid); + for (const pid of this.noReachableSince.keys()) if (!storeSet.has(pid)) this.noReachableSince.delete(pid); // Quarantine entries for peers gossip never mentions again would leak — drop // them once they are far past the window (re-entry from gossip self-cleans). const quarantineCutoff = now - 2 * UNREACHABLE_QUARANTINE_MS; @@ -1535,6 +1559,7 @@ export class Network { this._lastScores.clear(); this.redialBackoff.clear(); this.unreachableQuarantine.clear(); + this.noReachableSince.clear(); this.configuredPeerIDs.clear(); this.pxIngressLogKeys.clear(); if (this.node) { From d66ebf6c9dbe6f56dbfd09c81603689cc5739a0e Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 14:56:04 +0200 Subject: [PATCH 11/99] fix(network): shorten unreachable quarantine to sweep TTL --- backend/src/protocol/network.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index ad5d5981e..40ee0174b 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -101,8 +101,14 @@ const BOOTSTRAP_STATUS_STALE_MS = 30 * 60_000; * Gossip from nodes that still remember the dead peer keeps mentioning it; without * this window every mention would re-create its status row and burn a dial. Once * the window lapses a single probe is allowed again (self-heals on peer return). + * + * Kept equal to BOOTSTRAP_STATUS_STALE_MS deliberately: shorter would let stale + * gossip refresh rows faster than the sweep can expire them; longer would only + * delay re-discovery of a peer that genuinely came back (a returned peer that + * dials US escapes immediately via the peer:connect reset — this window matters + * only for peers that cannot initiate inbound connections). */ -const UNREACHABLE_QUARANTINE_MS = 60 * 60_000; +const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; /** * Maximum size (bytes) of an incoming pubsub payload we are willing to decode. * Our own control messages ride pubsub (WANT — tiny JSON), but older/foreign peers From b7830d681f702e8b0ecfaeeab2ca969921747af7 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:25:04 +0200 Subject: [PATCH 12/99] fix(network): compare destination identity in self-address skip --- backend/src/protocol/network.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 40ee0174b..37aad01c4 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1028,8 +1028,6 @@ export class Network { const myPeerID = this.node.peerId.toString(); const localCidrs = getLocalCidrs(); for (const peer of peers) { - // Skip our own address - if (peer.includes(myPeerID)) continue; try { const ma = Multiaddr(peer); // Safety net: refuse to add loopback / unreachable-private bootstrap @@ -1042,6 +1040,10 @@ export class Network { continue; } const peerID = extractDestinationPeerID(ma); + // Skip our own address — compare the DESTINATION identity, not the raw + // string: `/p2p//p2p-circuit/p2p/` contains our ID as the + // relay hop yet targets a remote peer and must not be dropped as self. + if (peerID === myPeerID) continue; if (peerID && origin === 'configured') this.configuredPeerIDs.add(peerID); // Skip peers recently evicted as unreachable — nodes that still remember // them keep gossiping their addrs, and without this window every mention From e306d43b9306195d09fbbbd7d86258283c7e0d61 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:25:38 +0200 Subject: [PATCH 13/99] fix(network): keep connected peer on identity-mismatch of one addr --- backend/src/protocol/network.ts | 34 ++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 37aad01c4..e7291cd73 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1096,14 +1096,34 @@ export class Network { } else { console.log(`⚠️ Could not connect to bootstrap peer (${kind}): ${peer} — ${message}`); } - // Crypto-verified identity mismatch ⇒ peerID stored in our peerStore - // is provably wrong for this address. Purge it so libp2p autodial - // stops retrying the dead identity. Safe because Noise handshake - // is unforgeable — a mismatch is definitive, never a transient - // network issue. Only triggers when we have an expected peerID - // to purge. + // Crypto-verified identity mismatch ⇒ THIS ADDRESS provably no longer + // belongs to the expected peer (Noise is unforgeable). It says nothing + // about the peer's other addresses: a peer healthy over a relay can + // still have one stale direct address that some other node now owns. + // So: peer alive through other connections → drop only the offending + // address; peer with no connections → full purge as before. if (kind === 'identity-mismatch' && peerID) { - await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); + const pid = peerIDFromString(peerID); + if (this.node.getConnections(pid).length > 0) { + const bare = peer.replace(/\/p2p\/[^/]+$/, ''); + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => { + const s = m.toString(); + return s !== peer && s !== bare; + }); + try { + const rec = await this.node.peerStore.get(pid); + const keep = rec.addresses.filter((a: any) => { + const s = a.multiaddr.toString(); + return s !== peer && s !== bare; + }); + if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); + } catch { + /* peer not in store — nothing to trim */ + } + console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${peer}`); + } else { + await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); + } // For DISCOVERED entries (peer-announce gossip), also drop the // status entry — there's no saved config row to "fix" and leaving // it visible just adds UI noise. For CONFIGURED entries, keep From 53230dbbade6b8c18f65ec931a005e996bf539fc Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:26:09 +0200 Subject: [PATCH 14/99] fix(network): restore peer state when purge races inbound connect --- backend/src/protocol/network.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index e7291cd73..1002108c3 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1184,6 +1184,22 @@ export class Network { } await this.node.peerStore.delete(pid); console.log(`[NET] purged stale peerStore entry ${peerID.slice(0, 16)}… (reason: ${reason})`); + // TOCTOU healing: an inbound connection can land between the caller's + // liveness check and the delete above. The peer:connect handler resets + // failure counters but cannot restore the bootstrap/keep-alive state this + // purge just removed — so if the peer is connected NOW, rebuild its dial + // state from the live connections; otherwise reconnect would silently die + // with the first drop. + const after = this.node.getConnections(pid); + if (after.length > 0) { + this.bootstrapPeerIDs.add(peerID); + this.unreachableQuarantine.delete(peerID); + await this.node.peerStore.merge(pid, { + multiaddrs: after.map(c => c.remoteAddr), + tags: { [KEEP_ALIVE]: { value: 1 } }, + }); + console.log(`[NET] purge raced an inbound connection — restored ${peerID.slice(0, 16)}…`); + } } catch (err: any) { trace(`[NET] purgeStalePeer ${peerID.slice(0, 16)} failed: ${err?.message ?? err}`); } From 5c9a2f1e0975db5f4d339c8407e44a01fe020467 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:27:28 +0200 Subject: [PATCH 15/99] fix(network): scope status tick state writes to current run epoch --- backend/src/protocol/network.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 1002108c3..3e927f93a 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -136,6 +136,14 @@ export class Network { private statusTickCount = 0; /** Guards against overlapping status ticks — see setupStatusInterval. */ private statusTickInFlight = false; + /** + * Lifecycle epoch, bumped by stop(). A status tick captures the epoch at + * entry and refuses to write per-peer state once it differs — an in-flight + * tick otherwise survives stop() and would repopulate freshly-cleared maps + * or purge peers of the NEXT node instance (whose configuredPeerIDs are not + * loaded yet) after a quick stop/start such as a factory reset. + */ + private runEpoch = 0; /** * Per-(peer,lish) timestamp of the last `have` response we sent. * Used to rate-limit responses to repeated `want` queries from the same peer for the same LISH: @@ -729,6 +737,7 @@ export class Network { // (and close connections of) a peer another tick just reconnected. if (this.statusTickInFlight) return; this.statusTickInFlight = true; + const epoch = this.runEpoch; try { const connectedPeers = this.node!.getPeers(); const allPeers = await this.node!.peerStore.all(); @@ -736,9 +745,12 @@ export class Network { dumpGossipsubScores({ node: this.node, pubsub: this.pubsub, settings: this.settings, lastScores: this._lastScores }, connectedPeers); // Periodic peer count refresh — catches cases where GRAFT/PRUNE events were missed this.checkPeerCounts(); - await this.runRedialMaintenance(connectedPeers, allPeers); + await this.runRedialMaintenance(connectedPeers, allPeers, epoch); + if (epoch !== this.runEpoch) return; await this.runZeroConnectionRecovery(connectedPeers); + if (epoch !== this.runEpoch) return; await this.maybePromotePeers(); + if (epoch !== this.runEpoch) return; // Fresh connection snapshot for the sweep — the tick-start snapshot is // stale by now: a discovered peer that reconnected during the re-dial // phase above must not have its status row swept as "not connected". @@ -755,7 +767,7 @@ export class Network { // churn at N≈100 without flooding logs or burning CPU on per-second probes. } - private async runRedialMaintenance(connectedPeers: any[], allPeers: any[]): Promise { + private async runRedialMaintenance(connectedPeers: any[], allPeers: any[], epoch: number = this.runEpoch): Promise { // Dial known peers not currently connected (maintains relay connections to NATed peers) const connectedSet = new Set(connectedPeers.map(p => p.toString())); const now = Date.now(); @@ -767,6 +779,7 @@ export class Network { let skippedNoReachable = 0; const localCidrs = getLocalCidrs(now); for (const peer of allPeers) { + if (epoch !== this.runEpoch) return; // stop() hit — this run's state is gone const pid = peer.id.toString(); if (connectedSet.has(pid)) { this.redialBackoff.delete(pid); // clear on observed connection @@ -817,6 +830,7 @@ export class Network { let idx = 0; const worker = async (): Promise => { while (idx < candidates.length) { + if (epoch !== this.runEpoch) return; // stop() hit — abandon remaining dials const c = candidates[idx++]!; console.debug(` ↻ Re-dial attempt peer=${c.pid} addrs=${c.addrSummary} fails=${c.failCount}`); try { @@ -844,6 +858,10 @@ export class Network { /* non-fatal */ } } catch (err: any) { + // A dial aborted by stop() looks like any other failure — do not let + // it repopulate maps that stop() just cleared, or evict against the + // NEXT node instance. + if (epoch !== this.runEpoch) return; // Exponential backoff: 30s × 2^failCount, capped at 10 min. const nextFailCount = c.failCount + 1; const delayMs = Math.min(30_000 * 2 ** c.failCount, 600_000); @@ -1567,6 +1585,7 @@ export class Network { } async stop(): Promise { + this.runEpoch++; // invalidate any in-flight status tick before touching state if (this.statusInterval) { clearInterval(this.statusInterval); this.statusInterval = null; From f9248c8ff2e285049cc37f16198362b24bc091bc Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:43:29 +0200 Subject: [PATCH 16/99] fix(network): guard promote and redial success writes by run epoch --- backend/src/protocol/network.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 3e927f93a..464dd2435 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -835,6 +835,9 @@ export class Network { console.debug(` ↻ Re-dial attempt peer=${c.pid} addrs=${c.addrSummary} fails=${c.failCount}`); try { await this.node!.dial(c.peer.id, { signal: AbortSignal.timeout(5000) }); + // Same guard as the failure path: a dial resolving after stop() must + // not write into the next run's state or the next node's peerStore. + if (epoch !== this.runEpoch) return; const conns = this.node!.getConnections(c.peer.id); const connDetail = conns .map(conn => { @@ -941,14 +944,14 @@ export class Network { } } - private async maybePromotePeers(): Promise { + private async maybePromotePeers(epoch: number = this.runEpoch): Promise { // Every 5th status tick (~150 s at 30 s status cadence) promote every // CONNECTED peer back to bootstrap priority (KEEP_ALIVE re-stamp + gossipsub // direct set). Disconnected peers are handled by runRedialMaintenance. this.statusTickCount++; if (this.statusTickCount % 5 === 0) { try { - await this.promoteKnownPeersToBootstrap(); + await this.promoteKnownPeersToBootstrap(epoch); } catch (err: any) { trace(`[NET] promoteKnownPeersToBootstrap failed: ${err?.message ?? err}`); } @@ -964,9 +967,13 @@ export class Network { * dead peers every promotion cycle and their permanent growth in the direct set. * Runs every ~150 s from the status tick. */ - private async promoteKnownPeersToBootstrap(): Promise { + private async promoteKnownPeersToBootstrap(epoch: number = this.runEpoch): Promise { if (!this.node) return; const allPeers = await this.node.peerStore.all(); + // stop() may have landed while peerStore.all() was pending — promoting now + // would repopulate bootstrap/tracker state the shutdown just cleared (or, + // after a fast restart, populate the NEXT node from the old snapshot). + if (epoch !== this.runEpoch) return; const myID = this.node.peerId.toString(); const connectedIDs = new Set(this.node.getPeers().map((p: any) => p.toString())); const maStrings: string[] = []; @@ -987,6 +994,7 @@ export class Network { if (maStrings.length > 0) { trace(`[NET] periodic autodial: promoting ${maStrings.length} connected peer(s) to bootstrap`); await this.addBootstrapPeers(maStrings); + if (epoch !== this.runEpoch) return; } // Also insert every connected peer into the gossipsub `direct` Set at runtime. // Direct peers have their own fast reconnect cadence (directConnectTicks × From ee444b37acd81687c04586b8f77e73bba2c9ad83 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:43:58 +0200 Subject: [PATCH 17/99] fix(network): merge only dial-verified addresses into peerstore --- backend/src/protocol/network.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 464dd2435..8bf6dda3b 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1100,10 +1100,12 @@ export class Network { const reuseExisting = alreadyKnown && peerID && this.node.getConnections(peerIDFromString(peerID)).length > 0; if (!reuseExisting) await this.node.dial(ma); if (peerID) { - await this.node.peerStore.merge(peerIDFromString(peerID), { - multiaddrs: [ma], - tags: { [KEEP_ALIVE]: { value: 1 } }, - }); + // Merge the address into peerStore ONLY when our own dial just + // verified it via Noise. On the reuseExisting path nothing proved + // this particular address belongs to the peer — merging it would + // let gossip poison a connected peer's address book with entries + // that later feed re-dials and can get the peer evicted. + await this.node.peerStore.merge(peerIDFromString(peerID), reuseExisting ? { tags: { [KEEP_ALIVE]: { value: 1 } } } : { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } }); } this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); console.log('✓ Connected to new bootstrap peer'); From 0ded60cf7741b27bf5cab59daa5d2ba3bbbbd167 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 15:44:37 +0200 Subject: [PATCH 18/99] fix(network): canonicalize address comparison in mismatch trimming --- backend/src/protocol/network.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 8bf6dda3b..fb75038eb 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1133,22 +1133,27 @@ export class Network { if (kind === 'identity-mismatch' && peerID) { const pid = peerIDFromString(peerID); if (this.node.getConnections(pid).length > 0) { - const bare = peer.replace(/\/p2p\/[^/]+$/, ''); + // Compare in CANONICAL form (parsed multiaddr toString) — the raw + // gossiped string may differ from what peerStore stored (e.g. + // expanded IPv6), and a non-matching filter would silently keep + // the poisoned address while logging that it was dropped. + const canonical = ma.toString(); + const canonicalBare = canonical.replace(/\/p2p\/[^/]+$/, ''); this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => { const s = m.toString(); - return s !== peer && s !== bare; + return s !== canonical && s !== canonicalBare; }); try { const rec = await this.node.peerStore.get(pid); const keep = rec.addresses.filter((a: any) => { const s = a.multiaddr.toString(); - return s !== peer && s !== bare; + return s !== canonical && s !== canonicalBare; }); if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); } catch { /* peer not in store — nothing to trim */ } - console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${peer}`); + console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${canonical}`); } else { await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); } From 29f9afdc4b0e5cafd9b1aa6e1de5e8457f1cd25e Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 16:32:46 +0200 Subject: [PATCH 19/99] fix(network): merge only newly-dialed addresses and guard intake by epoch --- backend/src/protocol/network.ts | 73 +++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index fb75038eb..993c5a4e8 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1053,7 +1053,13 @@ export class Network { } const myPeerID = this.node.peerId.toString(); const localCidrs = getLocalCidrs(); + // Fire-and-forget callers (peer-announce intake, startup joins) run outside + // the status-tick epoch guard. Capture the epoch so a dial that settles after + // a stop()/restart cannot record outcomes on the cleared tracker or write + // peerStore state for the NEXT node instance. + const epoch = this.runEpoch; for (const peer of peers) { + if (epoch !== this.runEpoch) return; try { const ma = Multiaddr(peer); // Safety net: refuse to add loopback / unreachable-private bootstrap @@ -1093,23 +1099,24 @@ export class Network { console.debug('Adding bootstrap peer:', peer); this.bootstrapTracker.markPending(networkID, peer, peerID, origin); try { - // Skip re-dialing when libp2p already has an active connection to this peer - // (typical when the same bootstrap entry appears in multiple lishnets). - // We still record the outcome so per-network status reflects "connected" - // rather than leaving the entry stuck at "pending". - const reuseExisting = alreadyKnown && peerID && this.node.getConnections(peerIDFromString(peerID)).length > 0; - if (!reuseExisting) await this.node.dial(ma); - if (peerID) { - // Merge the address into peerStore ONLY when our own dial just - // verified it via Noise. On the reuseExisting path nothing proved - // this particular address belongs to the peer — merging it would - // let gossip poison a connected peer's address book with entries - // that later feed re-dials and can get the peer evicted. - await this.node.peerStore.merge(peerIDFromString(peerID), reuseExisting ? { tags: { [KEEP_ALIVE]: { value: 1 } } } : { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } }); + // libp2p reuses an existing connection for dial(ma) WITHOUT contacting + // ma unless force:true. So a merge of ma is only "Noise-verified" when + // this call actually established a NEW connection — i.e. the peer had + // no connection before. If it was already connected (whether or not we + // tracked it as bootstrap), ma is unverified and must not enter the + // address book, or a topic subscriber could poison a connected peer's + // addresses with entries that later feed re-dials and cause eviction. + const pidObj = peerID ? peerIDFromString(peerID) : null; + const hadConnection = !!pidObj && this.node.getConnections(pidObj).length > 0; + if (!hadConnection) await this.node.dial(ma); + if (epoch !== this.runEpoch) return; + if (pidObj) { + await this.node.peerStore.merge(pidObj, hadConnection ? { tags: { [KEEP_ALIVE]: { value: 1 } } } : { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } }); } this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { + if (epoch !== this.runEpoch) return; const message = err?.message ?? String(err); const kind = classifyBootstrapError(message); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; @@ -1133,27 +1140,27 @@ export class Network { if (kind === 'identity-mismatch' && peerID) { const pid = peerIDFromString(peerID); if (this.node.getConnections(pid).length > 0) { - // Compare in CANONICAL form (parsed multiaddr toString) — the raw - // gossiped string may differ from what peerStore stored (e.g. - // expanded IPv6), and a non-matching filter would silently keep - // the poisoned address while logging that it was dropped. - const canonical = ma.toString(); + // Compare in a form that survives both multiaddr normalization + // (expanded → compressed IPv6) and DNS case / trailing-dot + // differences — otherwise a filter that fails to match silently + // keeps the poisoned address while logging that it was dropped. + const canonical = normalizeMultiaddrForCompare(ma.toString()); const canonicalBare = canonical.replace(/\/p2p\/[^/]+$/, ''); - this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => { - const s = m.toString(); - return s !== canonical && s !== canonicalBare; - }); + const matches = (s: string): boolean => { + const n = normalizeMultiaddrForCompare(s); + return n === canonical || n === canonicalBare; + }; + // Restrict the autodial-list filter to entries of THIS peer so a + // case-insensitive compare can never drop a different peer's addr. + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => extractDestinationPeerID(m) !== peerID || !matches(m.toString())); try { const rec = await this.node.peerStore.get(pid); - const keep = rec.addresses.filter((a: any) => { - const s = a.multiaddr.toString(); - return s !== canonical && s !== canonicalBare; - }); + const keep = rec.addresses.filter((a: any) => !matches(a.multiaddr.toString())); if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); } catch { /* peer not in store — nothing to trim */ } - console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${canonical}`); + console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${ma.toString()}`); } else { await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); } @@ -1692,6 +1699,18 @@ export class Network { * protection would target the wrong peer. The last /p2p/ component is always * the dial target. Returns null when the multiaddr carries no peer ID at all. */ +/** + * Normalize a multiaddr STRING for equality comparison. Multiaddr.toString() + * already compresses IPv6, but leaves DNS host case and trailing dots intact — + * `/dns4/EXAMPLE.COM./tcp/...` and `/dns4/example.com/tcp/...` address the same + * endpoint. Lowercasing is safe here because callers only ever compare addresses + * of the SAME peer, so a case-folded base58 peer-ID collision cannot drop a + * different peer's address. + */ +export function normalizeMultiaddrForCompare(s: string): string { + return s.toLowerCase().replace(/\.(?=\/|$)/g, ''); +} + export function extractDestinationPeerID(ma: any): string | null { try { const components: Array<{ code: number; value?: string }> = ma?.getComponents?.() ?? []; From bb1cd82277747305ab5a84a62dee7b948d9307f2 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 16:34:46 +0200 Subject: [PATCH 20/99] fix(settings): cap discovered rows and sweep by network membership --- backend/src/protocol/bootstrap-status.ts | 33 +++++++++++++--- backend/src/protocol/network.ts | 20 +++++++--- .../unit/protocol/bootstrap-status.test.ts | 38 +++++++++++++++++-- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 5dee81659..15945980a 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -1,5 +1,14 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialStatus, type BootstrapPeerOrigin } from '@shared'; +/** + * Hard ceiling on DISCOVERED (gossip-learned) rows kept per network. A hostile + * topic subscriber can announce unbounded unique addresses, all ending in one + * connected peer's ID; without a cap the tracker map, its snapshots, and the + * WebSocket updates grow without limit. Configured rows are never counted or + * dropped — they are finite user data. + */ +const MAX_DISCOVERED_PER_NETWORK = 256; + /** * Tracks per-network, per-bootstrap-peer dial outcome status. * @@ -48,6 +57,7 @@ export class BootstrapStatusTracker { const previous = net.get(multiaddr); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: new Date().toISOString() }); + this.capDiscovered(net); const snapshot = this.buildStatus(networkID); if (snapshot) this.onStatusChange?.(networkID, snapshot); } @@ -60,10 +70,20 @@ export class BootstrapStatusTracker { const previous = net.get(multiaddr); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString() }); + this.capDiscovered(net); const snapshot = this.buildStatus(networkID); if (snapshot) this.onStatusChange?.(networkID, snapshot); } + /** Bound discovered rows per network (drop the oldest) — see MAX_DISCOVERED_PER_NETWORK. */ + private capDiscovered(net: Map): void { + let discovered = 0; + for (const p of net.values()) if (p.origin === 'discovered') discovered++; + if (discovered <= MAX_DISCOVERED_PER_NETWORK) return; + const oldestFirst = [...net.entries()].filter(([, p]) => p.origin === 'discovered').sort((a, b) => Date.parse(a[1].updatedAt) - Date.parse(b[1].updatedAt)); + for (let i = 0; i < discovered - MAX_DISCOVERED_PER_NETWORK; i++) net.delete(oldestFirst[i]![0]); + } + /** Drop a single peer entry directly (used after identity-mismatch purge of discovered peers). */ deletePeer(networkID: string, multiaddr: string): void { const net = this.stats.get(networkID); @@ -97,18 +117,21 @@ export class BootstrapStatusTracker { /** * Drop discovered-origin entries that have gone stale: no status refresh within - * `ttlMs` AND no current connection to the peer. Dead peers stop being mentioned - * by gossip, so their rows stop refreshing and expire here — including rows - * frozen at 'connected' for a peer that silently died. Configured entries are + * `ttlMs` AND the peer is not an active member of THAT network. Dead peers stop + * being mentioned by gossip, so their rows stop refreshing and expire here — + * including rows frozen at 'connected' for a peer that silently died. The + * liveness predicate is scoped to the network (its topic subscribers), NOT the + * shared libp2p connection: a peer that left network B but is still connected + * through network A must not keep a stale row under B. Configured entries are * exempt (user data). `now` is injectable for tests. */ - sweepStale(ttlMs: number, isConnected: (peerID: string) => boolean, now: number = Date.now()): void { + sweepStale(ttlMs: number, isMember: (networkID: string, peerID: string) => boolean, now: number = Date.now()): void { for (const [networkID, peers] of [...this.stats]) { let changed = false; for (const [addr, p] of [...peers]) { if (p.origin !== 'discovered') continue; const pid = p.expectedPeerID ?? p.actualPeerID; - if (pid && isConnected(pid)) continue; + if (pid && isMember(networkID, pid)) continue; const updated = Date.parse(p.updatedAt); if (Number.isFinite(updated) && now - updated < ttlMs) continue; peers.delete(addr); diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 993c5a4e8..015dd0030 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -751,11 +751,21 @@ export class Network { if (epoch !== this.runEpoch) return; await this.maybePromotePeers(); if (epoch !== this.runEpoch) return; - // Fresh connection snapshot for the sweep — the tick-start snapshot is - // stale by now: a discovered peer that reconnected during the re-dial - // phase above must not have its status row swept as "not connected". - const connectedIDs = new Set(this.node!.getPeers().map((p: any) => p.toString())); - this.bootstrapTracker.sweepStale(BOOTSTRAP_STATUS_STALE_MS, pid => connectedIDs.has(pid)); + // Sweep by per-network membership (topic subscribers), not global + // connectivity: a peer that left this network but stays connected via + // another must still have its stale row here expire. Snapshot per topic + // lazily and freshly — the tick-start state is stale after the re-dial + // phase, and a peer that (re)subscribed during it must not be swept. + const topicMembers = new Map>(); + const isMember = (networkID: string, pid: string): boolean => { + let set = topicMembers.get(networkID); + if (!set) { + set = new Set(this.getTopicPeers(networkID)); + topicMembers.set(networkID, set); + } + return set.has(pid); + }; + this.bootstrapTracker.sweepStale(BOOTSTRAP_STATUS_STALE_MS, isMember); } catch (err: any) { trace(`[NET] statusInterval error: ${err?.message ?? err}`); } finally { diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index cbda29f9a..e6d7f9eaa 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -146,17 +146,17 @@ describe('BootstrapStatusTracker.sweepStale', () => { tracker.recordOutcome(NET, CONF_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'configured'); const past = Date.now() + TTL + 60_000; // both rows are then older than TTL - tracker.sweepStale(TTL, pid => pid === LIVE_ID, past); + tracker.sweepStale(TTL, (_net, pid) => pid === LIVE_ID, past); const addrs = tracker .getStatus(NET) ?.peers.map(p => p.multiaddr) .sort(); - // DEAD discovered row expired; LIVE row survives via connection; configured row untouchable. + // DEAD discovered row expired; LIVE row survives via membership; configured row untouchable. expect(addrs).toEqual([CONF_ADDR, LIVE_ADDR].sort()); }); - it('drops a row frozen at connected once the peer has no live connection', () => { + it('drops a row frozen at connected once the peer is no longer a network member', () => { const tracker = new BootstrapStatusTracker(); tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); @@ -165,7 +165,18 @@ describe('BootstrapStatusTracker.sweepStale', () => { expect(tracker.getStatus(NET)).toBe(null); }); - it('keeps rows within the TTL even without a connection', () => { + it('expires a row whose peer stays globally connected but left THIS network', () => { + // Membership predicate returns false for NET even though the peer is up + // elsewhere — the stale NET row must still expire past its TTL. + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, networkID => networkID !== NET, Date.now() + TTL + 60_000); + + expect(tracker.getStatus(NET)).toBe(null); + }); + + it('keeps rows within the TTL even for a non-member', () => { const tracker = new BootstrapStatusTracker(); tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); @@ -174,3 +185,22 @@ describe('BootstrapStatusTracker.sweepStale', () => { expect(tracker.getStatus(NET)?.peers.length).toBe(1); }); }); + +describe('BootstrapStatusTracker discovered-row cap', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWCapCapCapCapCapCapCapCapCapCapCapCapCapCapCA'; + + it('bounds discovered rows per network and keeps configured rows', () => { + const tracker = new BootstrapStatusTracker(); + // One configured row that must always survive. + tracker.recordOutcome(NET, `/ip4/198.51.100.1/tcp/9090/p2p/${PID}`, PID, 'connected', null, null, 'configured'); + // Flood well past the 256 cap with unique discovered addresses. + for (let i = 0; i < 400; i++) tracker.recordOutcome(NET, `/ip4/203.0.113.${i % 254}/tcp/${9000 + i}/p2p/${PID}`, PID, 'connected', null, null, 'discovered'); + + const peers = tracker.getStatus(NET)!.peers; + const discovered = peers.filter(p => p.origin === 'discovered').length; + const configured = peers.filter(p => p.origin === 'configured').length; + expect(discovered).toBeLessThanOrEqual(256); + expect(configured).toBe(1); + }); +}); From 7c2f903759dafa27aeaf2f69666b2fa8944ec11d Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 16:36:54 +0200 Subject: [PATCH 21/99] test(network): anchor unsubscribeTopic slice past new getTopicPeers use --- backend/tests/unit/protocol/network-mesh.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/protocol/network-mesh.test.ts b/backend/tests/unit/protocol/network-mesh.test.ts index 687ed4cab..c6d46f8e1 100644 --- a/backend/tests/unit/protocol/network-mesh.test.ts +++ b/backend/tests/unit/protocol/network-mesh.test.ts @@ -366,7 +366,11 @@ describe('subscribeTopic — peer count scheduling', () => { describe('unsubscribeTopic — peer count scheduling', () => { it('calls schedulePeerCountCheck immediately', () => { - const unsubBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('unsubscribeTopic(networkID: string)'), NETWORK_TS.indexOf('getTopicPeers')); + const unsubStart = NETWORK_TS.indexOf('unsubscribeTopic(networkID: string)'); + // Anchor the end marker AFTER the method start — getTopicPeers is now also + // referenced earlier (status-tick membership sweep), so a global indexOf + // would slice an empty range. + const unsubBlock = NETWORK_TS.slice(unsubStart, NETWORK_TS.indexOf('getTopicPeers', unsubStart)); expect(unsubBlock).toContain('this.schedulePeerCountCheck()'); }); }); From 41b74fb1bb8ac44d8ce78fd747a5142750904e4c Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 23 Jul 2026 17:06:09 +0200 Subject: [PATCH 22/99] fix(network): guard recordOutcome by run epoch after peerstore merge --- backend/src/protocol/network.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 015dd0030..744877def 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1123,6 +1123,10 @@ export class Network { if (pidObj) { await this.node.peerStore.merge(pidObj, hadConnection ? { tags: { [KEEP_ALIVE]: { value: 1 } } } : { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } }); } + // Re-check after the merge await too: stop() may have cleared the + // tracker while it was pending, and recordOutcome would otherwise + // resurrect a network row for the old (or next) node instance. + if (epoch !== this.runEpoch) return; this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { From b852a996414be24b3e5685cce388a053744b502c Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 6 Aug 2026 14:37:54 +0200 Subject: [PATCH 23/99] docs(network): attach the misplaced JSDoc blocks to their functions --- backend/src/protocol/network.ts | 36 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 744877def..aee43ff60 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1695,24 +1695,6 @@ export class Network { } } -/** - * Classify a libp2p dial error into a coarse status the UI can render distinctly. - * - * - `identity-mismatch`: the remote completed Noise handshake but reported a - * different peer ID than the multiaddr's `/p2p/` claimed. Always means - * the configured peerID is stale (or the address routes to a wrong node). - * - `timeout`: the dial never completed — peer offline, behind NAT without relay, - * firewall, or unreachable network path. - * - `error`: every other reason (invalid multiaddr, connection refused, protocol - * negotiation failure, etc). - */ -/** - * Extract the DESTINATION peer ID from a multiaddr. A circuit-relay address has - * the shape `/.../p2p//p2p-circuit/p2p/` — taking the FIRST - * /p2p/ component would return the relay's identity, so eviction and configured - * protection would target the wrong peer. The last /p2p/ component is always - * the dial target. Returns null when the multiaddr carries no peer ID at all. - */ /** * Normalize a multiaddr STRING for equality comparison. Multiaddr.toString() * already compresses IPv6, but leaves DNS host case and trailing dots intact — @@ -1725,6 +1707,13 @@ export function normalizeMultiaddrForCompare(s: string): string { return s.toLowerCase().replace(/\.(?=\/|$)/g, ''); } +/** + * Extract the DESTINATION peer ID from a multiaddr. A circuit-relay address has + * the shape `/.../p2p//p2p-circuit/p2p/` — taking the FIRST + * /p2p/ component would return the relay's identity, so eviction and configured + * protection would target the wrong peer. The last /p2p/ component is always + * the dial target. Returns null when the multiaddr carries no peer ID at all. + */ export function extractDestinationPeerID(ma: any): string | null { try { const components: Array<{ code: number; value?: string }> = ma?.getComponents?.() ?? []; @@ -1738,6 +1727,17 @@ export function extractDestinationPeerID(ma: any): string | null { return null; } +/** + * Classify a libp2p dial error into a coarse status the UI can render distinctly. + * + * - `identity-mismatch`: the remote completed Noise handshake but reported a + * different peer ID than the multiaddr's `/p2p/` claimed. Always means + * the configured peerID is stale (or the address routes to a wrong node). + * - `timeout`: the dial never completed — peer offline, behind NAT without relay, + * firewall, or unreachable network path. + * - `error`: every other reason (invalid multiaddr, connection refused, protocol + * negotiation failure, etc). + */ export function classifyBootstrapError(message: string): BootstrapPeerDialStatus { if (!message) return 'error'; if (message.includes('does not match expected remote identity key')) return 'identity-mismatch'; From 43fc567991531a3e2a49c2c673d4be937d0bdd72 Mon Sep 17 00:00:00 2001 From: LuRy Date: Thu, 6 Aug 2026 14:38:28 +0200 Subject: [PATCH 24/99] fix(network): scope the status-tick guard release to its own run --- backend/src/protocol/network.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index aee43ff60..8e712571e 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -769,7 +769,9 @@ export class Network { } catch (err: any) { trace(`[NET] statusInterval error: ${err?.message ?? err}`); } finally { - this.statusTickInFlight = false; + // Only release the guard for the run we belong to — after a stop()/start() + // the flag belongs to the new run, whose own tick may already hold it. + if (epoch === this.runEpoch) this.statusTickInFlight = false; } }, 30000); // Status interval 30 s. promoteKnownPeersToBootstrap + gossipsub.direct @@ -1626,6 +1628,11 @@ export class Network { clearInterval(this.statusInterval); this.statusInterval = null; } + // The epoch bump makes an in-flight tick bail out, but its `finally` runs + // asynchronously — a fast restart would otherwise find the flag still set and + // skip its own first tick. The tick only clears the flag for its own epoch, so + // clearing it here cannot be undone by the outgoing run. + this.statusTickInFlight = false; this.peerAnnounce.stop(); if (this.wantResponseCleanupInterval) { clearInterval(this.wantResponseCleanupInterval); From 349ff4cef5fcd804716531e9113f822b46df10dd Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 9 Aug 2026 14:37:06 +0200 Subject: [PATCH 25/99] fix(network): never evict peers while this node itself is offline --- backend/src/protocol/network.ts | 45 +++++++- .../tests/unit/protocol/peer-eviction.test.ts | 101 ++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 backend/tests/unit/protocol/peer-eviction.test.ts diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 8e712571e..e701b31a4 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -109,6 +109,33 @@ const BOOTSTRAP_STATUS_STALE_MS = 30 * 60_000; * only for peers that cannot initiate inbound connections). */ const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; + +/** + * Where the eviction window should run from after a re-dial failure. + * + * A failure only says something about the PEER when this node can reach anyone + * at all. While we are the disconnected one — laptop asleep, Wi-Fi off, VPN + * dropped — every dial fails, so the window is slid forward instead of + * accumulating. Without this, a local outage longer than REDIAL_EVICT_MIN_MS + * would evict the whole non-configured peerStore on the first dial after the + * connection came back. + */ +export function nextEvictionWindowStart(reachable: boolean, previous: number | undefined, now: number): number { + return reachable ? (previous ?? now) : now; +} + +/** + * Whether a run of re-dial failures has earned an eviction. + * + * Eviction is destructive — it purges the peerStore entry, drops the status row + * and quarantines the ID — so it needs all four conditions at once: we are + * demonstrably online, the peer has failed enough times, it has been failing for + * long enough, and it is not one the operator configured by hand. + */ +export function shouldEvictUnreachablePeer(input: { reachable: boolean; failCount: number; unreachableForMs: number; configured: boolean }): boolean { + if (!input.reachable || input.configured) return false; + return input.failCount >= REDIAL_EVICT_FAILS && input.unreachableForMs >= REDIAL_EVICT_MIN_MS; +} /** * Maximum size (bytes) of an incoming pubsub payload we are willing to decode. * Our own control messages ride pubsub (WANT — tiny JSON), but older/foreign peers @@ -880,7 +907,8 @@ export class Network { // Exponential backoff: 30s × 2^failCount, capped at 10 min. const nextFailCount = c.failCount + 1; const delayMs = Math.min(30_000 * 2 ** c.failCount, 600_000); - const firstFailure = this.redialBackoff.get(c.pid)?.firstFailure ?? Date.now(); + const reachable = this.hasConnectionOtherThan(c.peer.id); + const firstFailure = nextEvictionWindowStart(reachable, this.redialBackoff.get(c.pid)?.firstFailure, Date.now()); this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount, firstFailure }); console.debug(` ✗ Re-dial peer=${c.pid} failed: ${err.message ?? err} (tried: ${c.addrSummary}, next in ${Math.round(delayMs / 1000)}s)`); // Enough consecutive failures over enough time ⇒ the peer is gone, not @@ -890,7 +918,7 @@ export class Network { // quarantine the ID so gossip mentions don't immediately re-add it. // Configured bootstrap peers are exempt — user data, they must survive // any outage and keep their red status row instead. - if (nextFailCount >= REDIAL_EVICT_FAILS && Date.now() - firstFailure >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(c.pid)) { + if (shouldEvictUnreachablePeer({ reachable, failCount: nextFailCount, unreachableForMs: Date.now() - firstFailure, configured: this.configuredPeerIDs.has(c.pid) })) { // Last-moment liveness check: the peer may have connected (inbound // dial, another async path) while this worker was failing on stale // state. purgeStalePeer closes connections, so evicting here would @@ -1204,6 +1232,19 @@ export class Network { this.bootstrapTracker.setOnChange(cb); } + /** + * True when this node holds a live connection to somebody OTHER than the given + * peer. + * + * This is the difference between "that peer is gone" and "we are the ones who + * are offline", and eviction is only ever entitled to the first reading. The + * peer being judged is excluded because a connection to it would make the + * question moot — that case is handled separately, right before the purge. + */ + private hasConnectionOtherThan(peer: PeerID): boolean { + return !!this.node && this.node.getConnections().some(connection => !connection.remotePeer.equals(peer)); + } + /** * Remove a peerID from libp2p's peerStore + drop it from our bootstrap dedup set. * diff --git a/backend/tests/unit/protocol/peer-eviction.test.ts b/backend/tests/unit/protocol/peer-eviction.test.ts new file mode 100644 index 000000000..53b9db6e4 --- /dev/null +++ b/backend/tests/unit/protocol/peer-eviction.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'bun:test'; +import { nextEvictionWindowStart, shouldEvictUnreachablePeer } from '../../../src/protocol/network.ts'; + +/** The thresholds the production constants use, restated so a change to either is visible here. */ +const EVICT_FAILS = 6; +const EVICT_MIN_MS = 30 * 60_000; +const NOW = 1_700_000_000_000; + +describe('nextEvictionWindowStart', () => { + it('keeps the original start while this node can reach someone', () => { + const started = NOW - 20 * 60_000; + expect(nextEvictionWindowStart(true, started, NOW)).toBe(started); + }); + + it('starts the window at the first failure when there is no earlier one', () => { + expect(nextEvictionWindowStart(true, undefined, NOW)).toBe(NOW); + }); + + it('slides the window forward for every failure suffered while offline', () => { + // This is the whole point: an hour of failures during a local outage must + // not count towards the peer's unreachability. + const started = NOW - 60 * 60_000; + expect(nextEvictionWindowStart(false, started, NOW)).toBe(NOW); + }); +}); + +describe('shouldEvictUnreachablePeer', () => { + const gone = { reachable: true, failCount: EVICT_FAILS, unreachableForMs: EVICT_MIN_MS, configured: false }; + + it('evicts a peer that failed enough times over enough time', () => { + expect(shouldEvictUnreachablePeer(gone)).toBe(true); + }); + + it('never evicts while this node has no other connection', () => { + // A node that cannot reach anybody has no evidence about anybody. Without + // this, a laptop waking after an hour asleep would purge its whole peerStore. + expect(shouldEvictUnreachablePeer({ ...gone, reachable: false })).toBe(false); + }); + + it('never evicts a peer the operator configured by hand', () => { + // Configured bootstrap peers are user data: they keep their red status row + // through any outage rather than disappearing from the list. + expect(shouldEvictUnreachablePeer({ ...gone, configured: true })).toBe(false); + }); + + it('waits for both the count and the duration, not either', () => { + expect(shouldEvictUnreachablePeer({ ...gone, failCount: EVICT_FAILS - 1 })).toBe(false); + expect(shouldEvictUnreachablePeer({ ...gone, unreachableForMs: EVICT_MIN_MS - 1 })).toBe(false); + }); + + it('keeps evicting once both thresholds are exceeded', () => { + expect(shouldEvictUnreachablePeer({ ...gone, failCount: 50, unreachableForMs: 5 * EVICT_MIN_MS })).toBe(true); + }); +}); + +describe('the outage scenario the guard exists for', () => { + it('does not evict on the first failure after an hour offline', () => { + // Reproduces the reported defect: six failures accumulated while the host + // itself was offline, then the connection returns. Before the fix the stored + // window was an hour old, so this very first failure satisfied the duration + // test and purged the peer. + const windowStartedAt = NOW - 60 * 60_000; + + // Six failures during the outage: each one slides the window forward. + let start = windowStartedAt; + for (let failure = 1; failure <= EVICT_FAILS; failure++) start = nextEvictionWindowStart(false, start, NOW - (EVICT_FAILS - failure) * 1000); + + const firstFailureBackOnline = NOW; + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS + 1, unreachableForMs: firstFailureBackOnline - start, configured: false })).toBe(false); + }); + + it('still evicts a peer that stays unreachable while we are online', () => { + // The guard must not make eviction unreachable in practice — a genuinely + // dead peer is still removed once the failures accumulate with us online. + let start = nextEvictionWindowStart(true, undefined, NOW - EVICT_MIN_MS); + for (let failure = 2; failure <= EVICT_FAILS; failure++) start = nextEvictionWindowStart(true, start, NOW); + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS, unreachableForMs: NOW - start, configured: false })).toBe(true); + }); +}); + +describe('negative control', () => { + /** The pre-fix behaviour: the window ran from the first failure regardless of our own reachability. */ + function windowStartBeforeFix(previous: number | undefined, now: number): number { + return previous ?? now; + } + /** The pre-fix condition: count and duration only, with no notion of whether we were online. */ + function evictBeforeFix(failCount: number, unreachableForMs: number, configured: boolean): boolean { + return failCount >= EVICT_FAILS && unreachableForMs >= EVICT_MIN_MS && !configured; + } + + it('proves the scenario discriminates: the old logic evicts where the new one does not', () => { + // Same inputs as "does not evict on the first failure after an hour offline". + // If this assertion ever flips, the scenario stopped exercising the defect + // and the test above would pass for the wrong reason. + let start = NOW - 60 * 60_000; + for (let failure = 1; failure <= EVICT_FAILS; failure++) start = windowStartBeforeFix(start, NOW - (EVICT_FAILS - failure) * 1000); + + expect(evictBeforeFix(EVICT_FAILS + 1, NOW - start, false)).toBe(true); + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS + 1, unreachableForMs: NOW - nextEvictionWindowStart(false, start, NOW), configured: false })).toBe(false); + }); +}); From e14111d946d24bf962136d47953f5df494854b98 Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 09:11:09 +0200 Subject: [PATCH 26/99] fix(network): bind peer eviction and recovery dials to the run epoch --- backend/src/protocol/network.ts | 41 ++++++++---- .../protocol/peer-eviction-guards.test.ts | 67 +++++++++++++++++++ 2 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 backend/tests/unit/protocol/peer-eviction-guards.test.ts diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index e2e215151..35029ed32 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -857,9 +857,9 @@ export class Network { this.checkPeerCounts(); await this.runRedialMaintenance(connectedPeers, allPeers, epoch); if (epoch !== this.runEpoch) return; - await this.runZeroConnectionRecovery(connectedPeers); + await this.runZeroConnectionRecovery(connectedPeers, epoch); if (epoch !== this.runEpoch) return; - await this.maybePromotePeers(); + await this.maybePromotePeers(epoch); if (epoch !== this.runEpoch) return; // Sweep by per-network membership (topic subscribers), not global // connectivity: a peer that left this network but stays connected via @@ -1055,7 +1055,7 @@ export class Network { this.unreachableQuarantine.set(c.pid, Date.now()); this.redialBackoff.delete(c.pid); this.bootstrapTracker.deleteDiscoveredByPeerID(c.pid); - await this.purgeStalePeer(c.pid, `unreachable after ${nextFailCount} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`); + await this.purgeStalePeer(c.pid, `unreachable after ${nextFailCount} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`, epoch); } } } @@ -1079,7 +1079,9 @@ export class Network { for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } - private async runZeroConnectionRecovery(connectedPeers: any[]): Promise { + private async runZeroConnectionRecovery(connectedPeers: any[], epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; if (!AUTODIAL_WORKAROUND || connectedPeers.length !== 0 || this.bootstrapMultiaddrs.length === 0) return; console.log(` ⚠️ No connections - dialing ${this.bootstrapMultiaddrs.length} bootstrap peer(s) directly...`); // [NET-CHURN] dump: who left in the run-up to this zero-connection @@ -1105,9 +1107,12 @@ export class Network { const pid: string | undefined = p2pComponents.length > 0 ? p2pComponents[p2pComponents.length - 1].value : undefined; if (pid && this.isRedialSuppressed(pid)) continue; // deliberately left — don't resurrect it here const maStr = ma?.toString?.() ?? String(ma); + // Each dial awaits for up to 10s, so a stop() can land mid-loop; the + // remaining dials belong to a node this run no longer owns. + if (epoch !== this.runEpoch) return; try { console.log(` → Dialing ${maStr}`); - await this.node!.dial(ma, { signal: AbortSignal.timeout(10000) }); + await node.dial(ma, { signal: AbortSignal.timeout(10000) }); console.log(` ✓ Connected via ${maStr}`); break; } catch (err: any) { @@ -1351,7 +1356,7 @@ export class Network { } console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${ma.toString()}`); } else { - await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); + await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`, epoch); } // For DISCOVERED entries (peer-announce gossip), also drop the // status entry — there's no saved config row to "fix" and leaving @@ -1400,9 +1405,17 @@ export class Network { * * Best-effort: a peerStore.delete failure is logged at debug but does not throw — * the same peer will be re-purged next cycle if libp2p keeps trying it. + * + * `epoch` binds the call to the node instance it was started for. This is the most + * destructive path there is — it closes connections and deletes peerStore entries — + * and it awaits in the middle, so a stop()/start() landing between those awaits + * would otherwise let it finish against the NEXT node and evict a peer that + * instance never had a problem with. The node reference is captured once for the + * same reason: re-reading `this.node` after an await can hand back a different node. */ - async purgeStalePeer(peerID: string, reason: string): Promise { - if (!this.node) return; + async purgeStalePeer(peerID: string, reason: string, epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; this.bootstrapPeerIDs.delete(peerID); // Drop the peer's addrs from the autodial list too — this array is otherwise // push-only, so the zero-connection recovery loop would keep dialing addrs @@ -1416,7 +1429,7 @@ export class Network { try { const pid = peerIDFromString(peerID); // Drop existing connections so libp2p considers the entry fully gone. - const conns = this.node.getConnections(pid); + const conns = node.getConnections(pid); for (const c of conns) { try { await c.close(); @@ -1424,7 +1437,10 @@ export class Network { /* connection may already be closing */ } } - await this.node.peerStore.delete(pid); + // Closing connections yields; bail before the irreversible delete if this + // run no longer owns the node. + if (epoch !== this.runEpoch) return; + await node.peerStore.delete(pid); console.log(`[NET] purged stale peerStore entry ${peerID.slice(0, 16)}… (reason: ${reason})`); // TOCTOU healing: an inbound connection can land between the caller's // liveness check and the delete above. The peer:connect handler resets @@ -1432,11 +1448,12 @@ export class Network { // purge just removed — so if the peer is connected NOW, rebuild its dial // state from the live connections; otherwise reconnect would silently die // with the first drop. - const after = this.node.getConnections(pid); + if (epoch !== this.runEpoch) return; + const after = node.getConnections(pid); if (after.length > 0) { this.bootstrapPeerIDs.add(peerID); this.unreachableQuarantine.delete(peerID); - await this.node.peerStore.merge(pid, { + await node.peerStore.merge(pid, { multiaddrs: after.map(c => c.remoteAddr), tags: { [KEEP_ALIVE]: { value: 1 } }, }); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts new file mode 100644 index 000000000..d7f50db2e --- /dev/null +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered + * in peer-eviction.test.ts; what is exercised here is the part that actually closes + * connections and deletes peerStore entries, where the damage from getting it wrong is + * not a wrong boolean but a live peer evicted from the wrong libp2p node. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +describe('purgeStalePeer — epoch guard', () => { + function bareNetwork(onClose?: () => void) { + const deleted: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).node = { + getConnections: () => [ + { + async close(): Promise { + onClose?.(); + }, + }, + ], + peerStore: { + async delete(pid: { toString(): string }): Promise { + deleted.push(pid.toString()); + }, + async merge(): Promise {}, + }, + }; + return { network, deleted }; + } + + it('deletes the peerStore entry while the run still owns the node', async () => { + const { network, deleted } = bareNetwork(); + await (network as any).purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([PEER_ID]); + }); + + it('does not touch the peerStore when stop() lands while connections are closing', async () => { + // The exact race the epoch counter exists for: stop()/start() swaps in a new + // node during the close await, and the old run must not delete a peer from it. + const { network, deleted } = bareNetwork(); + const net = network as any; + net.node.getConnections = () => [ + { + async close(): Promise { + net.runEpoch++; // stop() during the await + }, + }, + ]; + await net.purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([]); + }); + + it('refuses to start at all for a stale epoch', async () => { + const { network, deleted } = bareNetwork(); + (network as any).runEpoch = 2; + await (network as any).purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([]); + }); +}); From bf6d7b439f76d436890d5151c2c1b720b8ff0271 Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 09:11:58 +0200 Subject: [PATCH 27/99] fix(network): require proof we are online before evicting an unreachable peer --- backend/src/protocol/network.ts | 19 +++-- .../protocol/peer-eviction-guards.test.ts | 77 +++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 35029ed32..b86e07b96 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -972,15 +972,24 @@ export class Network { // No dialable address ⇒ the failure counter can never fire for this // peer. Track how long it has been in this state; a disconnected peer // with zero reachable addrs for the whole eviction window is as gone - // as one that failed every dial. Same exemptions as the dial path. - const since = this.noReachableSince.get(pid) ?? now; - if (!this.noReachableSince.has(pid)) this.noReachableSince.set(pid, now); - if (now - since >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(pid)) { + // as one that failed every dial. + // + // "Unreachable" here means the dial gater rejects every stored address + // from where WE stand, which is not the same as the peer being gone: a + // peer reachable only over a LAN or VPN subnet stops passing the filter + // the moment that interface drops, through no fault of its own. So this + // path takes the same two safeguards as the dial-failure path below — + // evidence that we are online at all, and a liveness re-check right + // before acting — on top of the configured-peer exemption. + const weAreOnline = this.hasConnectionOtherThan(peer.id); + const since = nextEvictionWindowStart(weAreOnline, this.noReachableSince.get(pid), now); + this.noReachableSince.set(pid, since); + if (weAreOnline && now - since >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(pid) && this.node?.getConnections(peer.id).length === 0) { this.noReachableSince.delete(pid); this.unreachableQuarantine.set(pid, now); this.redialBackoff.delete(pid); this.bootstrapTracker.deleteDiscoveredByPeerID(pid); - await this.purgeStalePeer(pid, `no reachable addresses for ${Math.round((now - since) / 60_000)} min`); + await this.purgeStalePeer(pid, `no reachable addresses for ${Math.round((now - since) / 60_000)} min`, epoch); } continue; } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index d7f50db2e..9a5c1267f 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -9,6 +9,17 @@ import { Network } from '../../../src/protocol/network.ts'; */ const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +/** + * A peerStore entry with no addresses at all is the deterministic form of "nothing the + * dial gater will let us try" — using a real address would make the test depend on + * which subnets this host happens to be on. + */ +const NO_ADDRESSES: Array<{ multiaddr: { toString(): string } }> = []; + +function peerIdLike(id: string) { + return { toString: () => id, equals: (o: any) => String(o) === id }; +} + describe('purgeStalePeer — epoch guard', () => { function bareNetwork(onClose?: () => void) { const deleted: string[] = []; @@ -65,3 +76,69 @@ describe('purgeStalePeer — epoch guard', () => { expect(deleted).toEqual([]); }); }); + +/** + * A peer whose every stored address is rejected by the dial gater is not proof the peer + * is gone — a LAN/VPN-only peer looks exactly like this the moment our own interface + * drops. This path must therefore take the same self-online evidence as the + * dial-failure path, or a local outage evicts the whole non-configured peerStore. + */ +describe('runRedialMaintenance — eviction with no reachable address', () => { + function bareNetwork(opts: { weAreOnline: boolean; sinceMsAgo: number; configured?: boolean }) { + const purged: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); + (network as any).configuredPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { getConnections: () => [] }; + // Only the peer itself is ever asked about; "online" means we hold a connection + // to somebody else. + (network as any).hasConnectionOtherThan = () => opts.weAreOnline; + (network as any).purgeStalePeer = async (pid: string): Promise => { + purged.push(pid); + }; + return { network, purged }; + } + + const undialablePeer = { id: peerIdLike(PEER_ID), addresses: NO_ADDRESSES }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [undialablePeer], 1); + + it('evicts once the window has passed and we are demonstrably online', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000 }); + await run(network); + expect(purged).toEqual([PEER_ID]); + }); + + it('keeps the peer when the outage is ours', async () => { + const { network, purged } = bareNetwork({ weAreOnline: false, sinceMsAgo: 45 * 60_000 }); + await run(network); + expect(purged).toEqual([]); + }); + + it('slides the window forward during our outage instead of accumulating', async () => { + // Otherwise the first tick after connectivity returns would evict everything + // that went unreachable while we were down. + const { network } = bareNetwork({ weAreOnline: false, sinceMsAgo: 45 * 60_000 }); + await run(network); + const since = (network as any).noReachableSince.get(PEER_ID) as number; + expect(Date.now() - since).toBeLessThan(60_000); + }); + + it('never evicts a configured peer', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000, configured: true }); + await run(network); + expect(purged).toEqual([]); + }); + + it('keeps a peer that reconnected while the window was running', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000 }); + (network as any).node = { getConnections: () => [{}] }; // live again + await run(network); + expect(purged).toEqual([]); + }); +}); From 5551a0e3831c0a44e3c5ee2ab767579df1f998c4 Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 09:12:47 +0200 Subject: [PATCH 28/99] fix(network): verify a bootstrap address by the connection libp2p returns --- backend/src/protocol/network.ts | 27 +++++--- .../protocol/peer-eviction-guards.test.ts | 63 +++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index b86e07b96..5699b19ff 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1298,19 +1298,26 @@ export class Network { console.debug('Adding bootstrap peer:', peer); this.bootstrapTracker.markPending(networkID, peer, peerID, origin); try { - // libp2p reuses an existing connection for dial(ma) WITHOUT contacting - // ma unless force:true. So a merge of ma is only "Noise-verified" when - // this call actually established a NEW connection — i.e. the peer had - // no connection before. If it was already connected (whether or not we - // tracked it as bootstrap), ma is unverified and must not enter the - // address book, or a topic subscriber could poison a connected peer's - // addresses with entries that later feed re-dials and cause eviction. + // Always hand the address to libp2p and let IT decide whether a dial is + // needed. Skipping the call whenever any connection to the peer existed + // was too coarse: libp2p reuses only a DIRECT, unlimited connection, and + // deliberately dials when it holds a relayed one and the new address + // would upgrade it to direct. Pre-empting that cost us the upgrade, and + // left a bad configured address permanently untested — its identity + // mismatch undiscovered — whenever the peer happened to be reachable + // some other way. + // + // The address may still enter the address book only when it is + // Noise-verified, otherwise a topic subscriber could poison a connected + // peer's addresses with entries that later feed re-dials and eviction. + // Verification is now read off the RESULT: the connection libp2p handed + // back is proof for `ma` only if that is the address it is actually on. const pidObj = peerID ? peerIDFromString(peerID) : null; - const hadConnection = !!pidObj && this.node.getConnections(pidObj).length > 0; - if (!hadConnection) await this.node.dial(ma); + const conn = await this.node.dial(ma); + const verifiedThisAddr = normalizeMultiaddrForCompare(String(conn?.remoteAddr ?? '')).startsWith(normalizeMultiaddrForCompare(ma.toString().replace(/\/p2p\/[^/]+$/, ''))); if (epoch !== this.runEpoch) return; if (pidObj) { - await this.node.peerStore.merge(pidObj, hadConnection ? { tags: { [KEEP_ALIVE]: { value: 1 } } } : { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } }); + await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); } // Re-check after the merge await too: stop() may have cleared the // tracker while it was pending, and recordOutcome would otherwise diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 9a5c1267f..345d86b02 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -142,3 +142,66 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { expect(purged).toEqual([]); }); }); + +/** + * An address may enter the address book only once libp2p has actually connected over + * it. Deciding that up-front from "do we have any connection to this peer" was too + * coarse — it skipped relay→direct upgrades and left bad addresses untested — so the + * answer is now read off the connection libp2p returns. + */ +describe('addBootstrapPeers — only a verified address enters the peerStore', () => { + function bareNetwork(remoteAddrOfReturnedConn: string) { + const merges: Array> = []; + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + return { remoteAddr: { toString: () => remoteAddrOfReturnedConn } }; + }, + peerStore: { + async merge(_pid: unknown, patch: Record): Promise { + merges.push(patch); + }, + }, + }; + return { network, merges, dialled }; + } + + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + it('dials the address even when the peer is already connected', async () => { + // libp2p reuses a suitable connection by itself and dials when the new address + // would upgrade a relayed one; pre-empting that lost the upgrade. + const { network, dialled } = bareNetwork(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`); + (network as any).node.getConnections = () => [{}]; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR]); + }); + + it('stores the address when the connection is actually on it', async () => { + const { network, merges } = bareNetwork(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(merges).toHaveLength(1); + expect(merges[0]).toHaveProperty('multiaddrs'); + }); + + it('withholds the address when libp2p answered over a different one', async () => { + // Reused/relayed connection: this address was never contacted, so it is not + // Noise-verified and must not be poisonable into the address book. + const { network, merges } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p-circuit/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(merges).toHaveLength(1); + expect(merges[0]).not.toHaveProperty('multiaddrs'); + }); +}); From 611845be4da2f6f3009714b11a6dc5468abacec0 Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 13:39:39 +0200 Subject: [PATCH 29/99] fix(network): compare the whole dial endpoint instead of a string prefix --- backend/src/protocol/network.ts | 17 ++++++++- .../protocol/peer-eviction-guards.test.ts | 37 ++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 5699b19ff..cb0469550 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1314,7 +1314,7 @@ export class Network { // back is proof for `ma` only if that is the address it is actually on. const pidObj = peerID ? peerIDFromString(peerID) : null; const conn = await this.node.dial(ma); - const verifiedThisAddr = normalizeMultiaddrForCompare(String(conn?.remoteAddr ?? '')).startsWith(normalizeMultiaddrForCompare(ma.toString().replace(/\/p2p\/[^/]+$/, ''))); + const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); if (epoch !== this.runEpoch) return; if (pidObj) { await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); @@ -2133,6 +2133,21 @@ export function normalizeMultiaddrForCompare(s: string): string { return s.toLowerCase().replace(/\.(?=\/|$)/g, ''); } +/** + * Whether two multiaddrs denote the same transport endpoint, ignoring a trailing + * `/p2p/` (a dial target usually carries it, `Connection.remoteAddr` may not). + * + * Compares the WHOLE remaining address, never a prefix: `/ip4/x/tcp/80` is a string + * prefix of `/ip4/x/tcp/8080`, so prefix matching would accept a connection on one + * port as proof for another — exactly the unverified-address case this is used to + * reject. + */ +export function isSameDialEndpoint(a: string, b: string): boolean { + const strip = (s: string): string => normalizeMultiaddrForCompare(s).replace(/\/p2p\/[^/]+$/, ''); + const left = strip(a); + return left.length > 0 && left === strip(b); +} + /** * Extract the DESTINATION peer ID from a multiaddr. A circuit-relay address has * the shape `/.../p2p//p2p-circuit/p2p/` — taking the FIRST diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 345d86b02..8298a0a36 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'bun:test'; -import { Network } from '../../../src/protocol/network.ts'; +import { Network, isSameDialEndpoint } from '../../../src/protocol/network.ts'; /** * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered @@ -205,3 +205,38 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( expect(merges[0]).not.toHaveProperty('multiaddrs'); }); }); + +/** + * The address-equality test behind "was THIS address verified". It decides whether an + * unverified address may enter the peerStore, so a false positive is a security bug, + * not a cosmetic one. + */ +describe('isSameDialEndpoint', () => { + const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + + it('accepts the same endpoint with and without the peer id suffix', () => { + expect(isSameDialEndpoint('/ip4/203.0.113.4/tcp/9090', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(true); + }); + + it('rejects a different port that merely shares a prefix', () => { + // /tcp/80 is a string prefix of /tcp/8080 — prefix matching would call a + // connection on 8080 proof that the claimed port 80 works. + expect(isSameDialEndpoint(`/ip4/203.0.113.4/tcp/8080/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/80/p2p/${PEER_A}`)).toBe(false); + }); + + it('rejects a different host', () => { + expect(isSameDialEndpoint(`/ip4/203.0.113.5/tcp/9090/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); + + it('rejects a relayed connection as proof of a direct address', () => { + expect(isSameDialEndpoint(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_A}/p2p-circuit/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); + + it('ignores DNS case and a trailing dot', () => { + expect(isSameDialEndpoint('/dns4/EXAMPLE.COM./tcp/443', '/dns4/example.com/tcp/443')).toBe(true); + }); + + it('treats a missing connection address as no proof', () => { + expect(isSameDialEndpoint('', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); +}); From 81ec9760d0600c50c4011d1287b75beb148e8f3e Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 13:40:37 +0200 Subject: [PATCH 30/99] fix(network): probe a configured bootstrap address instead of reusing a connection --- backend/src/protocol/network.ts | 12 ++++- .../protocol/peer-eviction-guards.test.ts | 51 ++++++++++++++++++- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index cb0469550..ae057f0a9 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1312,8 +1312,18 @@ export class Network { // peer's addresses with entries that later feed re-dials and eviction. // Verification is now read off the RESULT: the connection libp2p handed // back is proof for `ma` only if that is the address it is actually on. + // A configured address is the user's own claim and its status row is how + // they debug it, so it gets a real probe: `force` makes libp2p contact + // THIS address instead of handing back a connection it already holds to + // the same peer, which is what let a broken configured entry sit there + // showing "connected" — and kept its identity mismatch undiscovered — + // merely because the peer was reachable some other way. + // + // Discovered addresses never force: they arrive from gossip, and a peer + // that names many of them could otherwise make us open a connection per + // address. For those, libp2p's own reuse is the desired behaviour. const pidObj = peerID ? peerIDFromString(peerID) : null; - const conn = await this.node.dial(ma); + const conn = await this.node.dial(ma, origin === 'configured' ? { force: true } : {}); const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); if (epoch !== this.runEpoch) return; if (pidObj) { diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 8298a0a36..cf39be352 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -152,6 +152,7 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { describe('addBootstrapPeers — only a verified address enters the peerStore', () => { function bareNetwork(remoteAddrOfReturnedConn: string) { const merges: Array> = []; + const forced: boolean[] = []; const dialled: string[] = []; const network = Object.create(Network.prototype) as Network; (network as any).runEpoch = 1; @@ -165,8 +166,9 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], - async dial(ma: { toString(): string }): Promise { + async dial(ma: { toString(): string }, opts?: { force?: boolean }): Promise { dialled.push(ma.toString()); + forced.push(opts?.force === true); return { remoteAddr: { toString: () => remoteAddrOfReturnedConn } }; }, peerStore: { @@ -175,7 +177,7 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( }, }, }; - return { network, merges, dialled }; + return { network, merges, dialled, forced }; } const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; @@ -240,3 +242,48 @@ describe('isSameDialEndpoint', () => { expect(isSameDialEndpoint('', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); }); }); + +/** + * A configured bootstrap address is the user's own claim and its status row is how they + * debug it, so it must be probed for real rather than satisfied by any connection that + * happens to exist to the same peer. Gossiped addresses must not force: a peer naming + * many of them could otherwise make us open a connection per address. + */ +describe('addBootstrapPeers — forced probe only for configured addresses', () => { + function bareNetwork() { + const forced: boolean[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [{}], // already connected to this peer some other way + async dial(ma: { toString(): string }, opts?: { force?: boolean }): Promise { + forced.push(opts?.force === true); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, forced }; + } + + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + it('forces the dial for a configured address', async () => { + const { network, forced } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(forced).toEqual([true]); + }); + + it('does not force the dial for a discovered address', async () => { + const { network, forced } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(forced).toEqual([false]); + }); +}); From 47d9e4532c1c2bddeb27f6af4881366e16e7130a Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 14:34:42 +0200 Subject: [PATCH 31/99] fix(network): end the eviction exemption when a peer leaves the config --- backend/src/protocol/network.ts | 38 ++++++------- .../unit/protocol/network-disconnect.test.ts | 3 +- .../protocol/peer-eviction-guards.test.ts | 55 ++++++++++++++++++- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index ae057f0a9..f324f5a47 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -177,7 +177,7 @@ export class Network { * Lifecycle epoch, bumped by stop(). A status tick captures the epoch at * entry and refuses to write per-peer state once it differs — an in-flight * tick otherwise survives stop() and would repopulate freshly-cleared maps - * or purge peers of the NEXT node instance (whose configuredPeerIDs are not + * or purge peers of the NEXT node instance (whose configured peers are not * loaded yet) after a quick stop/start such as a factory reset. */ private runEpoch = 0; @@ -200,6 +200,15 @@ export class Network { * separate from bootstrapPeerIDs, which also collects peer-announce * discoveries: those are plain content peers and must remain * disconnectable by lishnet leave (isBootstrapOrRelayPeer). + * + * This is also what exempts a peer from unreachable-eviction: configured + * entries are user data, so a bootstrap hub that is down for half an hour must + * keep its peerStore entry and its addrs instead of being purged. Both + * questions — "is this infrastructure?" and "may we evict it?" — are the same + * question about the same fact, so they read the same set. Keeping two sets for + * it meant they could disagree, and they did: only one of them was ever pruned, + * so a peer the user had already removed from the config stayed eviction-exempt + * until restart. */ private configuredBootstrapPeerIDs: Set = new Set(); private dcutrPeers: Set = new Set(); @@ -286,20 +295,6 @@ export class Network { * peerStore/bootstrap sets until maxPeerAge while every tick re-scans them. */ private readonly noReachableSince = new Map(); - /** - * Peer IDs that appear in at least one network's CONFIGURED bootstrap list. - * These are user data — the unreachable-eviction path must never purge them, - * or a bootstrap hub that is down for half an hour would lose its peerStore - * entry and its addrs in bootstrapMultiaddrs until the next restart. - * - * Grow-only by design: entries are not removed when a bootstrap row is - * deleted or its network disabled, so a formerly-configured peer stays - * eviction-exempt until restart. That errs on the safe side (a peer is - * merely redialed longer than necessary); per-network refcounting would be - * required to shrink it correctly and is not worth the bookkeeping. - */ - private readonly configuredPeerIDs = new Set(); - /** * Peers deliberately hung up by {@link disconnectPeer} (leave-network), keyed by * the lishnet they were left with. Redial maintenance / discovery must NOT @@ -984,7 +979,7 @@ export class Network { const weAreOnline = this.hasConnectionOtherThan(peer.id); const since = nextEvictionWindowStart(weAreOnline, this.noReachableSince.get(pid), now); this.noReachableSince.set(pid, since); - if (weAreOnline && now - since >= REDIAL_EVICT_MIN_MS && !this.configuredPeerIDs.has(pid) && this.node?.getConnections(peer.id).length === 0) { + if (weAreOnline && now - since >= REDIAL_EVICT_MIN_MS && !this.configuredBootstrapPeerIDs.has(pid) && this.node?.getConnections(peer.id).length === 0) { this.noReachableSince.delete(pid); this.unreachableQuarantine.set(pid, now); this.redialBackoff.delete(pid); @@ -1052,7 +1047,7 @@ export class Network { // quarantine the ID so gossip mentions don't immediately re-add it. // Configured bootstrap peers are exempt — user data, they must survive // any outage and keep their red status row instead. - if (shouldEvictUnreachablePeer({ reachable, failCount: nextFailCount, unreachableForMs: Date.now() - firstFailure, configured: this.configuredPeerIDs.has(c.pid) })) { + if (shouldEvictUnreachablePeer({ reachable, failCount: nextFailCount, unreachableForMs: Date.now() - firstFailure, configured: this.configuredBootstrapPeerIDs.has(c.pid) })) { // Last-moment liveness check: the peer may have connected (inbound // dial, another async path) while this worker was failing on stale // state. purgeStalePeer closes connections, so evicting here would @@ -1268,7 +1263,6 @@ export class Network { // relay hop yet targets a remote peer and must not be dropped as self. if (peerID === myPeerID) continue; if (peerID && origin === 'configured') { - this.configuredPeerIDs.add(peerID); this.configuredBootstrapPeerIDs.add(peerID); // A re-configured bootstrap peer means its network was (re-)joined — it // is no longer "left", so lift any redial suppression left by a prior @@ -1510,7 +1504,11 @@ export class Network { * lishnet layer when a bootstrap entry is removed from config or belongs only * to a lishnet being left, so `isBootstrapOrRelayPeer` stops treating a peer * that is no longer configured (nor shared with another joined network) as - * infrastructure that leave-network must keep connected. + * infrastructure that leave-network must keep connected — and so the + * unreachable-eviction exemption ends with it. + * + * Both callers already establish that the peer is configured in NO joined + * network before calling, so this needs no refcount of its own. */ pruneConfiguredBootstrapPeer(peerID: string): void { this.configuredBootstrapPeerIDs.delete(peerID); @@ -2094,7 +2092,7 @@ export class Network { this.redialBackoff.clear(); this.unreachableQuarantine.clear(); this.noReachableSince.clear(); - this.configuredPeerIDs.clear(); + this.configuredBootstrapPeerIDs.clear(); this.redialSuppressedByNet.clear(); this.pxIngressLogKeys.clear(); if (this.node) { diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index ccecd4ab1..322af41cb 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -125,7 +125,7 @@ describe('Network.runRedialMaintenance — leave-peer suppression', () => { (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map(); - (network as any).configuredPeerIDs = new Set(); + (network as any).configuredBootstrapPeerIDs = new Set(); // A reconnected peer's suppression is lifted only if it currently shares a joined // topic — model that via a pubsub whose subscribers list the "back on topic" peers. (network as any).pubsub = { @@ -217,7 +217,6 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); (network as any).configuredBootstrapPeerIDs = new Set(); - (network as any).configuredPeerIDs = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index cf39be352..c94cf7003 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -92,7 +92,7 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { (network as any).redialSuppressedByNet = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); - (network as any).configuredPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; @@ -158,7 +158,6 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); - (network as any).configuredPeerIDs = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -256,7 +255,6 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); - (network as any).configuredPeerIDs = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -287,3 +285,54 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () expect(forced).toEqual([false]); }); }); + +/** + * "Is this configured infrastructure?" and "may we auto-evict it?" are the same + * question about the same fact. They used to be answered from two separate sets and + * only one of them was ever pruned, so a peer the user had already deleted from the + * bootstrap config kept its eviction exemption until the process restarted. + */ +describe('configured exemption ends when the peer leaves the config', () => { + function bareNetwork() { + const purged: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - 45 * 60_000]]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { getConnections: () => [] }; + (network as any).hasConnectionOtherThan = () => true; + (network as any).purgeStalePeer = async (pid: string): Promise => { + purged.push(pid); + }; + return { network, purged }; + } + + const undialable = { id: peerIdLike(PEER_ID), addresses: NO_ADDRESSES }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [undialable], 1); + + it('protects the peer while it is still configured', async () => { + const { network, purged } = bareNetwork(); + await run(network); + expect(purged).toEqual([]); + }); + + it('stops protecting it once the config entry is gone', async () => { + const { network, purged } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + await run(network); + expect(purged).toEqual([PEER_ID]); + }); + + it('drops the infrastructure status in the same step', async () => { + // The two used to be able to disagree; pruning must settle both at once. + const { network } = bareNetwork(); + expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(true); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(false); + }); +}); From 96c8cfaa4cb458df8c8744bdb2391dbf71ec4b6f Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 19:27:25 +0200 Subject: [PATCH 32/99] fix(network): keep discovered participants when the bootstrap config changes --- backend/src/protocol/bootstrap-status.ts | 15 ++++-- .../unit/protocol/bootstrap-status.test.ts | 47 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 15945980a..67168e34b 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -143,13 +143,22 @@ export class BootstrapStatusTracker { } } - /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ + /** + * Drop bootstrap status entries no longer in the configured peer list (after an update). + * + * `keepMultiaddrs` is the network's CONFIGURED list, so only configured rows may be + * judged by it. Discovered rows are not in it and never will be — deleting them here + * would clear the participant list of everything gossip has found, on nothing more + * than a bootstrap edit or a "refresh from public list", until gossip happens to + * mention each peer again. Discovered rows leave via their own paths: the staleness + * sweep, the per-network cap, or eviction of the peer ID. + */ pruneEntries(networkID: string, keepMultiaddrs: string[]): void { const peers = this.stats.get(networkID); if (!peers) return; const keep = new Set(keepMultiaddrs); - for (const addr of [...peers.keys()]) { - if (!keep.has(addr)) peers.delete(addr); + for (const [addr, peer] of [...peers.entries()]) { + if (peer.origin === 'configured' && !keep.has(addr)) peers.delete(addr); } if (peers.size === 0) this.stats.delete(networkID); const snapshot = this.buildStatus(networkID); diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index e6d7f9eaa..d3e9b1b28 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -204,3 +204,50 @@ describe('BootstrapStatusTracker discovered-row cap', () => { expect(configured).toBe(1); }); }); + +/** + * pruneEntries is fed the network's CONFIGURED bootstrap list after the user edits it. + * Since this tracker also holds gossip-discovered rows, judging every row by that list + * would empty the participant view on a bootstrap edit — including a "refresh from + * public list" — leaving it blank until gossip mentions each peer again. + */ +describe('BootstrapStatusTracker.pruneEntries', () => { + const NET = 'netAAAA'; + const CONF_KEPT = '/ip4/192.0.2.1/tcp/9090/p2p/12D3KooWConfKeptKeptKeptKeptKeptKeptKeptKeptKeptK'; + const CONF_DROPPED = '/ip4/192.0.2.2/tcp/9090/p2p/12D3KooWConfGoneGoneGoneGoneGoneGoneGoneGoneGone'; + const DISCOVERED = '/ip4/192.0.2.3/tcp/9090/p2p/12D3KooWDiscDiscDiscDiscDiscDiscDiscDiscDiscDis'; + + function seeded(): BootstrapStatusTracker { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, CONF_KEPT, null, 'connected', null, null, 'configured'); + tracker.recordOutcome(NET, CONF_DROPPED, null, 'connected', null, null, 'configured'); + tracker.recordOutcome(NET, DISCOVERED, null, 'connected', null, null, 'discovered'); + return tracker; + } + + const addresses = (tracker: BootstrapStatusTracker): string[] => (tracker.getStatus(NET)?.peers ?? []).map(p => p.multiaddr); + + it('drops a configured row that left the config', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).not.toContain(CONF_DROPPED); + }); + + it('keeps a configured row that is still in the config', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).toContain(CONF_KEPT); + }); + + it('keeps discovered rows, which the configured list never mentions', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).toContain(DISCOVERED); + }); + + it('keeps discovered rows even when the whole config is cleared', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, []); + expect(addresses(tracker)).toEqual([DISCOVERED]); + }); +}); From 7dda2aeb6cc58a0a834f68885f435ca5d805bac5 Mon Sep 17 00:00:00 2001 From: LuRy Date: Fri, 14 Aug 2026 19:27:25 +0200 Subject: [PATCH 33/99] fix(network): count only online failures towards peer eviction --- backend/src/protocol/network.ts | 27 +++++++++--- .../tests/unit/protocol/peer-eviction.test.ts | 43 ++++++++++++++++++- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index f324f5a47..7d3de7a18 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -134,6 +134,21 @@ export function nextEvictionWindowStart(reachable: boolean, previous: number | u return reachable ? (previous ?? now) : now; } +/** + * How many failures count TOWARDS EVICTION after another one. + * + * Counted separately from the backoff's failCount, which must keep growing through a + * local outage so we stop hammering the dialer. Eviction asks a different question — + * "has the PEER failed us N times?" — and a dial attempted while we had no connectivity + * answers nothing, so the run resets. Without this, two hours offline bank enough + * failures that the peer is evicted after the window even though only a couple of + * genuine failures happened once we were back: the backoff caps at 10 minutes, so a + * 30-minute window holds barely three attempts. + */ +export function nextEvictionFailCount(reachable: boolean, previous: number | undefined): number { + return reachable ? (previous ?? 0) + 1 : 0; +} + /** * Whether a run of re-dial failures has earned an eviction. * @@ -285,7 +300,7 @@ export class Network { * capped at 10 min), so a persistently-unreachable peer does not saturate the * re-dial pool every 30s. Successful dial clears the entry. */ - private readonly redialBackoff = new Map(); + private readonly redialBackoff = new Map(); /** peerID → eviction time. Blocks re-adding a just-evicted unreachable peer from gossip for UNREACHABLE_QUARANTINE_MS. */ private readonly unreachableQuarantine = new Map(); /** @@ -1037,8 +1052,10 @@ export class Network { const nextFailCount = c.failCount + 1; const delayMs = Math.min(30_000 * 2 ** c.failCount, 600_000); const reachable = this.hasConnectionOtherThan(c.peer.id); - const firstFailure = nextEvictionWindowStart(reachable, this.redialBackoff.get(c.pid)?.firstFailure, Date.now()); - this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount, firstFailure }); + const previous = this.redialBackoff.get(c.pid); + const firstFailure = nextEvictionWindowStart(reachable, previous?.firstFailure, Date.now()); + const evictionFails = nextEvictionFailCount(reachable, previous?.evictionFails); + this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount, firstFailure, evictionFails }); console.debug(` ✗ Re-dial peer=${c.pid} failed: ${err.message ?? err} (tried: ${c.addrSummary}, next in ${Math.round(delayMs / 1000)}s)`); // Enough consecutive failures over enough time ⇒ the peer is gone, not // flaky. The dial above went by peer ID, so libp2p tried EVERY known @@ -1047,7 +1064,7 @@ export class Network { // quarantine the ID so gossip mentions don't immediately re-add it. // Configured bootstrap peers are exempt — user data, they must survive // any outage and keep their red status row instead. - if (shouldEvictUnreachablePeer({ reachable, failCount: nextFailCount, unreachableForMs: Date.now() - firstFailure, configured: this.configuredBootstrapPeerIDs.has(c.pid) })) { + if (shouldEvictUnreachablePeer({ reachable, failCount: evictionFails, unreachableForMs: Date.now() - firstFailure, configured: this.configuredBootstrapPeerIDs.has(c.pid) })) { // Last-moment liveness check: the peer may have connected (inbound // dial, another async path) while this worker was failing on stale // state. purgeStalePeer closes connections, so evicting here would @@ -1059,7 +1076,7 @@ export class Network { this.unreachableQuarantine.set(c.pid, Date.now()); this.redialBackoff.delete(c.pid); this.bootstrapTracker.deleteDiscoveredByPeerID(c.pid); - await this.purgeStalePeer(c.pid, `unreachable after ${nextFailCount} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`, epoch); + await this.purgeStalePeer(c.pid, `unreachable after ${evictionFails} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`, epoch); } } } diff --git a/backend/tests/unit/protocol/peer-eviction.test.ts b/backend/tests/unit/protocol/peer-eviction.test.ts index 53b9db6e4..53a50961d 100644 --- a/backend/tests/unit/protocol/peer-eviction.test.ts +++ b/backend/tests/unit/protocol/peer-eviction.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'bun:test'; -import { nextEvictionWindowStart, shouldEvictUnreachablePeer } from '../../../src/protocol/network.ts'; +import { nextEvictionWindowStart, nextEvictionFailCount, shouldEvictUnreachablePeer } from '../../../src/protocol/network.ts'; /** The thresholds the production constants use, restated so a change to either is visible here. */ const EVICT_FAILS = 6; @@ -99,3 +99,44 @@ describe('negative control', () => { expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS + 1, unreachableForMs: NOW - nextEvictionWindowStart(false, start, NOW), configured: false })).toBe(false); }); }); + +/** + * The backoff counter and the eviction counter answer different questions. The backoff + * must keep growing through a local outage so we stop hammering the dialer; eviction + * asks whether the PEER failed us, and a dial attempted with no connectivity of our own + * answers nothing. + */ +describe('nextEvictionFailCount', () => { + it('counts a failure that happened while we were online', () => { + expect(nextEvictionFailCount(true, 2)).toBe(3); + }); + + it('starts the count at one for a first online failure', () => { + expect(nextEvictionFailCount(true, undefined)).toBe(1); + }); + + it('resets the run when the failure happened during our own outage', () => { + expect(nextEvictionFailCount(false, 5)).toBe(0); + }); +}); + +describe('an outage must not bank failures towards eviction', () => { + /** Backoff caps at 10 min, so a 30 min window holds roughly three attempts. */ + const ONLINE_ATTEMPTS_IN_WINDOW = 3; + + it('does not evict on three online failures after a long offline run', () => { + let evictionFails: number | undefined; + for (let i = 0; i < 20; i++) evictionFails = nextEvictionFailCount(false, evictionFails); // hours offline + for (let i = 0; i < ONLINE_ATTEMPTS_IN_WINDOW; i++) evictionFails = nextEvictionFailCount(true, evictionFails); + + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: evictionFails!, unreachableForMs: 45 * 60_000, configured: false })).toBe(false); + }); + + it('still evicts once the peer really has failed enough times while we were online', () => { + let evictionFails: number | undefined; + for (let i = 0; i < 20; i++) evictionFails = nextEvictionFailCount(false, evictionFails); + for (let i = 0; i < 6; i++) evictionFails = nextEvictionFailCount(true, evictionFails); + + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: evictionFails!, unreachableForMs: 45 * 60_000, configured: false })).toBe(true); + }); +}); From 739cd3848d11b514e7c6c1349ed00434a9f5885b Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 00:02:39 +0200 Subject: [PATCH 34/99] fix(network): fold only DNS host case in multiaddr compare --- backend/src/protocol/network.ts | 13 ++++--- .../protocol/peer-eviction-guards.test.ts | 36 ++++++++++++++++++- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 7d3de7a18..a13a0b274 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -2148,14 +2148,17 @@ export class Network { /** * Normalize a multiaddr STRING for equality comparison. Multiaddr.toString() - * already compresses IPv6, but leaves DNS host case and trailing dots intact — + * already compresses IPv6, but leaves DNS host case and the FQDN root dot intact — * `/dns4/EXAMPLE.COM./tcp/...` and `/dns4/example.com/tcp/...` address the same - * endpoint. Lowercasing is safe here because callers only ever compare addresses - * of the SAME peer, so a case-folded base58 peer-ID collision cannot drop a - * different peer's address. + * endpoint. + * + * Only the HOST of a DNS component is folded, never the whole address. A circuit + * multiaddr carries `/p2p/` in the middle, and a base58 peer ID is + * case-significant — case-folding an identifier is a different question from + * case-folding a hostname, and this function is only entitled to the second. */ export function normalizeMultiaddrForCompare(s: string): string { - return s.toLowerCase().replace(/\.(?=\/|$)/g, ''); + return s.replace(/\/(dns|dns4|dns6|dnsaddr)\/([^/]+)/gi, (_match, protocol: string, host: string) => `/${protocol.toLowerCase()}/${host.toLowerCase().replace(/\.+$/, '')}`); } /** diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index c94cf7003..ed5ea6825 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'bun:test'; -import { Network, isSameDialEndpoint } from '../../../src/protocol/network.ts'; +import { Network, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; /** * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered @@ -240,6 +240,40 @@ describe('isSameDialEndpoint', () => { it('treats a missing connection address as no proof', () => { expect(isSameDialEndpoint('', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); }); + + it('rejects a relay whose identity differs only in case', () => { + // Only the trailing /p2p/ is stripped, so a circuit address is compared with + // the RELAY's peer id still in the middle. Folding the whole string would make two + // distinct relays look like one endpoint. + const relay = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const otherRelay = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrnGS17fo'; + expect(isSameDialEndpoint(`/ip4/198.51.100.1/tcp/4001/p2p/${relay}/p2p-circuit/p2p/${PEER_A}`, `/ip4/198.51.100.1/tcp/4001/p2p/${otherRelay}/p2p-circuit/p2p/${PEER_A}`)).toBe(false); + }); +}); + +/** + * Case folding is a hostname question, not an identifier question. DNS is defined as + * case-insensitive; a base58 peer id is not, and the two live in the same string. + */ +describe('normalizeMultiaddrForCompare', () => { + it('folds DNS host case and drops the FQDN root dot', () => { + expect(normalizeMultiaddrForCompare('/dns4/EXAMPLE.COM./tcp/443')).toBe('/dns4/example.com/tcp/443'); + }); + + it('folds every DNS protocol variant', () => { + expect(normalizeMultiaddrForCompare('/dnsaddr/Bootstrap.Example.COM/tcp/443')).toBe('/dnsaddr/bootstrap.example.com/tcp/443'); + expect(normalizeMultiaddrForCompare('/dns6/Example.COM/tcp/443')).toBe('/dns6/example.com/tcp/443'); + }); + + it('leaves a peer id untouched', () => { + const peer = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + expect(normalizeMultiaddrForCompare(`/dns4/EXAMPLE.COM/tcp/443/p2p/${peer}`)).toBe(`/dns4/example.com/tcp/443/p2p/${peer}`); + }); + + it('leaves an address with no DNS component completely alone', () => { + const addr = '/ip4/203.0.113.4/tcp/9090/p2p/12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + expect(normalizeMultiaddrForCompare(addr)).toBe(addr); + }); }); /** From fe62d3757202ba9d3adfa28237e47187ab73e77e Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 09:04:21 +0200 Subject: [PATCH 35/99] fix(network): abandon bootstrap dials for a superseded network config --- backend/src/lishnet/lishnets.ts | 4 +++ backend/src/protocol/network.ts | 57 ++++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index ab42cc684..0ac6398b7 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -208,6 +208,10 @@ export class Networks { this.network.unsubscribeTopic(id); this.joinedNetworks.delete(id); + // Abandon any bootstrap job still walking this network's list. Left half-way + // through, it would keep dialing peers of a network we just left and clear the + // redial suppression the loop below is about to apply. + this.network.bumpBootstrapGeneration(id); // Subscribers of any OTHER joined lishnet must stay connected (shared // infrastructure). Compute this set BEFORE the bootstrap cleanup so that loop diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index a13a0b274..491aa28af 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -228,6 +228,12 @@ export class Network { private configuredBootstrapPeerIDs: Set = new Set(); private dcutrPeers: Set = new Set(); private bootstrapMultiaddrs: any[] = []; + /** + * Per-network bootstrap-config version, bumped on every replace / reset / leave. + * Read by {@link addBootstrapPeers} so a job started for a superseded list stops + * instead of re-adding entries that are no longer configured. + */ + private readonly bootstrapGeneration: Map = new Map(); // Topic handlers: topic -> Set of handler functions private topicHandlers: Map> = new Map(); @@ -1260,9 +1266,19 @@ export class Network { // the status-tick epoch guard. Capture the epoch so a dial that settles after // a stop()/restart cannot record outcomes on the cleared tracker or write // peerStore state for the NEXT node instance. + // + // The generation covers the other axis: the node stays up but THIS network's + // bootstrap list is replaced, reset or left while we are part-way down it. The + // loop dials sequentially and one dial can take seconds, so an old job would + // otherwise keep walking the old list — re-adding entries the user has just + // removed and re-marking them configured, which exempts them from the stale + // sweep until restart. Exactly the resurrection this eviction work exists to + // prevent. const epoch = this.runEpoch; + const generation = this.bootstrapGenerationOf(networkID); + const superseded = (): boolean => epoch !== this.runEpoch || generation !== this.bootstrapGenerationOf(networkID); for (const peer of peers) { - if (epoch !== this.runEpoch) return; + if (superseded()) return; try { const ma = Multiaddr(peer); // Safety net: refuse to add loopback / unreachable-private bootstrap @@ -1336,18 +1352,30 @@ export class Network { const pidObj = peerID ? peerIDFromString(peerID) : null; const conn = await this.node.dial(ma, origin === 'configured' ? { force: true } : {}); const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); - if (epoch !== this.runEpoch) return; + if (superseded()) return; if (pidObj) { await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); } // Re-check after the merge await too: stop() may have cleared the // tracker while it was pending, and recordOutcome would otherwise // resurrect a network row for the old (or next) node instance. - if (epoch !== this.runEpoch) return; + if (superseded()) return; + // `force: true` defeats connection REUSE, but not a dial to the same peer + // ID already in libp2p's queue: this call joins that job and can be handed + // the connection its other address won. For a CONFIGURED entry the row + // means "this address works", so an unverified dial must leave it pending + // rather than turn it green — a wrong address that the peer happens to + // survive through another route is exactly what the row exists to expose. + // Discovered rows carry the weaker "the peer answered" meaning and are + // recorded either way. + if (origin === 'configured' && !verifiedThisAddr) { + trace(`[NET] bootstrap addr unverified (connection came back on another address), left pending: ${peer}`); + continue; + } this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { - if (epoch !== this.runEpoch) return; + if (superseded()) return; const message = err?.message ?? String(err); const kind = classifyBootstrapError(message); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; @@ -1716,14 +1744,34 @@ export class Network { /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ pruneBootstrapStatus(networkID: string, keepMultiaddrs: string[]): void { + this.bumpBootstrapGeneration(networkID); this.bootstrapTracker.pruneEntries(networkID, keepMultiaddrs); } /** Reset the bootstrap status for a single network (used when re-joining). */ resetBootstrapStatus(networkID: string): void { + this.bumpBootstrapGeneration(networkID); this.bootstrapTracker.resetNetwork(networkID); } + /** + * Current bootstrap-config version of a network. Entries with no network (the + * startup catalog dial) share version 0 and are only bound by the run epoch. + */ + private bootstrapGenerationOf(networkID: string | null): number { + return networkID === null ? 0 : (this.bootstrapGeneration.get(networkID) ?? 0); + } + + /** + * Declare a network's configured bootstrap list superseded, abandoning any + * `addBootstrapPeers` job still walking the previous one. Called whenever that + * list is replaced, reset or left — see the generation comment in + * {@link addBootstrapPeers} for what an unabandoned job would resurrect. + */ + bumpBootstrapGeneration(networkID: string): void { + this.bootstrapGeneration.set(networkID, this.bootstrapGenerationOf(networkID) + 1); + } + // ========================================================================= // Topic (lishnet) management // ========================================================================= @@ -2104,6 +2152,7 @@ export class Network { this.bootstrapPeerIDs.clear(); this.bootstrapTracker.clear(); this.bootstrapMultiaddrs = []; + this.bootstrapGeneration.clear(); this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); From 96874b8d6b0876a0a88550ec048b1b951346bd04 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 09:04:21 +0200 Subject: [PATCH 36/99] fix(network): emit an empty participant list when the last row goes --- backend/src/protocol/bootstrap-status.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 67168e34b..9a3c1b82e 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -161,8 +161,11 @@ export class BootstrapStatusTracker { if (peer.origin === 'configured' && !keep.has(addr)) peers.delete(addr); } if (peers.size === 0) this.stats.delete(networkID); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + // Emit the empty list rather than nothing when the last row goes: buildStatus + // returns null for a dropped network, and skipping the callback would leave the + // UI showing the very row that was just removed. Same fallback the other + // removal paths use. + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); } /** Reset the bootstrap status for a single network (used when re-joining). */ From 282617017e2437a1e613e2f29d872b9fbb641f85 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 09:04:21 +0200 Subject: [PATCH 37/99] test(network): cover config supersede, unverified row and empty emit --- .../tests/unit/lishnet/leave-network.test.ts | 6 ++ .../unit/protocol/bootstrap-status.test.ts | 23 ++++ .../unit/protocol/network-disconnect.test.ts | 2 + .../protocol/peer-eviction-guards.test.ts | 101 +++++++++++++++++- 4 files changed, 130 insertions(+), 2 deletions(-) diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 9299f3982..abc004843 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -24,6 +24,8 @@ interface MockNet { isBootstrapOrRelayPeer(pid: string): boolean; disconnectPeer(pid: string, networkID: string): Promise; pruneConfiguredBootstrapPeer(pid: string): void; + bumpBootstrapGeneration(networkID: string): void; + generationBumps: string[]; clearRedialSuppressionForNetwork(networkID: string): void; suppressionClearedFor: string[]; } @@ -38,6 +40,7 @@ function makeMockNet(): MockNet { bootstrapOrRelay: new Set(), prunedBootstrap: [], suppressionClearedFor: [], + generationBumps: [], getTopicPeers(id) { return this.topicPeers.get(id) ?? []; }, @@ -61,6 +64,9 @@ function makeMockNet(): MockNet { pruneConfiguredBootstrapPeer(pid) { this.prunedBootstrap.push(pid); }, + bumpBootstrapGeneration(networkID) { + this.generationBumps.push(networkID); + }, clearRedialSuppressionForNetwork(networkID) { this.suppressionClearedFor.push(networkID); }, diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index d3e9b1b28..8f61aadfc 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -250,4 +250,27 @@ describe('BootstrapStatusTracker.pruneEntries', () => { tracker.pruneEntries(NET, []); expect(addresses(tracker)).toEqual([DISCOVERED]); }); + + /** + * Removing the last row drops the whole network from the tracker, at which point + * buildStatus has nothing to return. Staying silent there would leave the UI + * rendering the very row that was just deleted, so the empty list is emitted + * explicitly — the same fallback the other removal paths use. + */ + it('emits an empty list when the last remaining row is removed', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, CONF_KEPT, null, 'connected', null, null, 'configured'); + const seen: Array<{ networkID: string; peers: unknown[] }> = []; + tracker.setOnChange((networkID, status) => seen.push({ networkID, peers: status.peers })); + tracker.pruneEntries(NET, []); + expect(seen).toEqual([{ networkID: NET, peers: [] }]); + }); + + it('still emits the surviving rows when only some are removed', () => { + const tracker = seeded(); + const seen: string[][] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.map(p => p.multiaddr))); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(seen).toEqual([[CONF_KEPT, DISCOVERED]]); + }); }); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index 322af41cb..d2fc16f27 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -19,6 +19,7 @@ function makeNetwork() { const deleted: string[] = []; const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map>(); + (network as any).bootstrapGeneration = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).redialBackoff = new Map(); @@ -218,6 +219,7 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapGeneration = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index ed5ea6825..6aaaf31f2 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -161,7 +161,14 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapGeneration = new Map(); + const outcomes: string[] = []; + (network as any).bootstrapTracker = { + markPending() {}, + recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { + outcomes.push(status); + }, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -176,7 +183,7 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( }, }, }; - return { network, merges, dialled, forced }; + return { network, merges, dialled, forced, outcomes }; } const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; @@ -197,6 +204,30 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( expect(merges[0]).toHaveProperty('multiaddrs'); }); + /** + * `force: true` defeats connection reuse but not a dial to the same peer id already + * in libp2p's queue — this call joins that job and can be handed the connection its + * OTHER address won. A configured row means "this address works", so it must not go + * green on a connection that never touched it. + */ + it('leaves a configured row pending when the connection came back on another address', async () => { + const { network, outcomes } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(outcomes).toEqual([]); + }); + + it('still records a discovered row, whose status only claims the peer answered', async () => { + const { network, outcomes } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(outcomes).toEqual(['connected']); + }); + + it('records the configured row once the connection is on the address itself', async () => { + const { network, outcomes } = bareNetwork(ADDR); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(outcomes).toEqual(['connected']); + }); + it('withholds the address when libp2p answered over a different one', async () => { // Reused/relayed connection: this address was never contacted, so it is not // Noise-verified and must not be poisonable into the address book. @@ -292,6 +323,7 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -370,3 +402,68 @@ describe('configured exemption ends when the peer leaves the config', () => { expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(false); }); }); + +/** + * The bootstrap list is walked one peer at a time and a single dial can take seconds. + * If the user edits that list — or leaves the network — mid-walk, the job started for + * the OLD list must stop: carrying on would re-add entries that are no longer + * configured AND re-mark them configured, which exempts them from the stale sweep for + * the rest of the process's life. That is precisely the resurrection this work exists + * to prevent, so the guard is checked here against a real second list entry. + */ +describe('addBootstrapPeers — superseded bootstrap configuration', () => { + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function bareNetwork(onFirstDial?: (network: Network) => void) { + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + // Model the edit landing while the FIRST dial is still in flight. + if (dialled.length === 1) onFirstDial?.(network); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { + async merge(): Promise {}, + }, + }; + return { network, dialled }; + } + + it('abandons the rest of the list when the network configuration is superseded', async () => { + const { network, dialled } = bareNetwork(n => n.bumpBootstrapGeneration('net-a')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A]); + }); + + it('does not re-mark the abandoned entry as configured', async () => { + const { network } = bareNetwork(n => n.bumpBootstrapGeneration('net-a')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_B)).toBe(false); + }); + + it('walks the whole list when nothing supersedes it', async () => { + const { network, dialled } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A, ADDR_B]); + }); + + it('is not disturbed by an edit to a DIFFERENT network', async () => { + const { network, dialled } = bareNetwork(n => n.bumpBootstrapGeneration('net-other')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A, ADDR_B]); + }); +}); From c6d7ffc4d586714fcd9d74ed6c2c97f1f5f81c42 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 12:10:05 +0200 Subject: [PATCH 38/99] fix(lishnets): apply bootstrap list edits to the running node --- backend/src/lishnet/lishnets.ts | 50 +++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 0ac6398b7..604ac6e6d 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -388,7 +388,17 @@ export class Networks { } update(network: LISHNetworkConfig): boolean { - return updateLISHnet(this.db, network); + const existing = this.get(network.networkID); + const ok = updateLISHnet(this.db, network); + // The general edit form carries the bootstrap list as well, so this path can + // change it just like updateBootstrapPeers does. Without the same runtime + // synchronisation the edit would reach only the database and the live node + // would keep dialing the previous list until restart. + if (!ok || !existing) return ok; + const previous = Networks.cleanBootstrapList(existing.bootstrapPeers); + const cleaned = Networks.cleanBootstrapList(network.bootstrapPeers ?? []); + if (previous.join('\n') !== cleaned.join('\n')) this.syncBootstrapRuntime(network.networkID, existing.bootstrapPeers, cleaned); + return ok; } async delete(id: string): Promise { @@ -435,24 +445,42 @@ export class Networks { async updateBootstrapPeers(id: string, bootstrapPeers: string[]): Promise { const existing = this.get(id); if (!existing) return null; - const cleaned = bootstrapPeers.filter(p => typeof p === 'string' && p.trim().length > 0); - // Drop the bootstrap-exemption for peer IDs removed from this network's - // config, unless still configured for another joined network. Prevents a - // removed bootstrap entry from lingering as infrastructure that a later - // leave-network would refuse to disconnect. + const cleaned = Networks.cleanBootstrapList(bootstrapPeers); + const next: LISHNetworkConfig = { ...existing, bootstrapPeers: cleaned }; + updateLISHnet(this.db, next); + this.syncBootstrapRuntime(id, existing.bootstrapPeers, cleaned); + return next; + } + + /** Drop blank entries from a user-supplied bootstrap list. */ + private static cleanBootstrapList(peers: string[]): string[] { + return peers.filter(p => typeof p === 'string' && p.trim().length > 0); + } + + /** + * Bring the running node in line with a network's new configured bootstrap list. + * + * Shared by the bootstrap-only editor and the general network edit form, because + * both can change that list and a change that reaches only the database leaves the + * live node working from the previous one until restart. + * + * Three things have to happen: peer IDs that left the list lose their + * bootstrap-exemption (unless another joined network still configures them, else a + * removed entry lingers as infrastructure a later leave refuses to disconnect), + * the status rows are pruned — which also invalidates any bootstrap job still + * walking the old list — and the new entries are dialed. + */ + private syncBootstrapRuntime(id: string, previousPeers: string[], cleaned: string[]): void { const nextIDs = new Set(Networks.bootstrapPeerIDsOf(cleaned)); const elsewhere = this.configuredBootstrapPeerIDsElsewhere(id); - for (const pid of Networks.bootstrapPeerIDsOf(existing.bootstrapPeers)) { + for (const pid of Networks.bootstrapPeerIDsOf(previousPeers)) { if (!nextIDs.has(pid) && !elsewhere.has(pid)) this.network.pruneConfiguredBootstrapPeer(pid); } - const next: LISHNetworkConfig = { ...existing, bootstrapPeers: cleaned }; - updateLISHnet(this.db, next); this.network.pruneBootstrapStatus(id, cleaned); if (this.joinedNetworks.has(id) && cleaned.length > 0) { this.network.addBootstrapPeers(cleaned, id, 'configured').catch(err => { - console.error(`[Networks] re-dial after updateBootstrapPeers failed:`, err?.message ?? err); + console.error(`[Networks] bootstrap re-dial after config change failed:`, err?.message ?? err); }); } - return next; } } From 384bae4b91e99b377c965e520e39668b6c5c6ade Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 12:10:05 +0200 Subject: [PATCH 39/99] fix(network): close a bootstrap dial that lands after leave-network --- backend/src/protocol/network.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 491aa28af..7d7140518 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1352,6 +1352,16 @@ export class Network { const pidObj = peerID ? peerIDFromString(peerID) : null; const conn = await this.node.dial(ma, origin === 'configured' ? { force: true } : {}); const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); + // A dial already in flight cannot be called back: hangUp only closes + // connections that ALREADY exist, so a leave-network landing mid-dial finds + // nothing to close and this connection surfaces a moment after the cleanup + // finished. Abandoning the loop would leave it open, so close it here — the + // suppression set is what says the user deliberately left this peer. + if (peerID && networkID && this.isRedialSuppressed(peerID)) { + trace(`[NET] bootstrap dial landed after leave, disconnecting: ${peerID.slice(0, 16)}`); + await this.disconnectPeer(peerID, networkID); + return; + } if (superseded()) return; if (pidObj) { await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); From 70e4fd24320e56cefb9e41ef81e8612948216634 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sat, 15 Aug 2026 12:10:06 +0200 Subject: [PATCH 40/99] test(network): cover form-edited bootstrap list and post-leave dial --- .../tests/unit/lishnet/leave-network.test.ts | 72 +++++++++++++++++++ .../protocol/peer-eviction-guards.test.ts | 51 +++++++++++++ 2 files changed, 123 insertions(+) diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index abc004843..9a9133967 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -1,4 +1,6 @@ import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { initLISHnetsTables, addLISHnet } from '../../../src/db/lishnets.ts'; import { Networks } from '../../../src/lishnet/lishnets.ts'; /** @@ -26,6 +28,10 @@ interface MockNet { pruneConfiguredBootstrapPeer(pid: string): void; bumpBootstrapGeneration(networkID: string): void; generationBumps: string[]; + pruneBootstrapStatus(networkID: string, keep: string[]): void; + prunedStatus: Array<{ networkID: string; keep: string[] }>; + addBootstrapPeers(peers: string[], networkID: string, origin: string): Promise; + dialledLists: Array<{ networkID: string; peers: string[]; origin: string }>; clearRedialSuppressionForNetwork(networkID: string): void; suppressionClearedFor: string[]; } @@ -41,6 +47,8 @@ function makeMockNet(): MockNet { prunedBootstrap: [], suppressionClearedFor: [], generationBumps: [], + prunedStatus: [], + dialledLists: [], getTopicPeers(id) { return this.topicPeers.get(id) ?? []; }, @@ -67,6 +75,12 @@ function makeMockNet(): MockNet { bumpBootstrapGeneration(networkID) { this.generationBumps.push(networkID); }, + pruneBootstrapStatus(networkID, keep) { + this.prunedStatus.push({ networkID, keep }); + }, + async addBootstrapPeers(peers, networkID, origin) { + this.dialledLists.push({ networkID, peers, origin }); + }, clearRedialSuppressionForNetwork(networkID) { this.suppressionClearedFor.push(networkID); }, @@ -278,3 +292,61 @@ describe('Networks.joinNetwork — onNetworkJoined notification', () => { expect(net.suppressionClearedFor).toEqual(['net-a']); }); }); + +/** + * The bootstrap list can be edited from two screens: the participants view, which + * calls updateBootstrapPeers, and the ordinary "edit network" form, which calls + * update. Only the first used to reach the running node, so a list changed through + * the form was written to the database and then ignored until restart — the node + * kept dialing the peers the user had just removed. + */ +describe('Networks.update — a changed bootstrap list reaches the running node', () => { + const NET = 'net-a'; + const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function seeded(bootstrapPeers: string[]) { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers, enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).network = mock; + (networks as any).db = db; + (networks as any).joinedNetworks = new Set([NET]); + return { networks, mock }; + } + + const edit = (networks: Networks, bootstrapPeers: string[]): boolean => (networks as any).update({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled: true, created: '2026-01-01T00:00:00.000Z' }); + + it('prunes the status and dials the new list when the entries change', () => { + const { networks, mock } = seeded([ADDR_A]); + edit(networks, [ADDR_B]); + expect(mock.prunedStatus).toEqual([{ networkID: NET, keep: [ADDR_B] }]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_B], origin: 'configured' }]); + }); + + it('drops the bootstrap exemption of a peer removed through the form', () => { + const { networks, mock } = seeded([ADDR_A, ADDR_B]); + edit(networks, [ADDR_A]); + expect(mock.prunedBootstrap).toEqual([PEER_B]); + }); + + it('leaves the running node alone when only the name changed', () => { + const { networks, mock } = seeded([ADDR_A]); + edit(networks, [ADDR_A]); + expect(mock.prunedStatus).toEqual([]); + expect(mock.dialledLists).toEqual([]); + expect(mock.prunedBootstrap).toEqual([]); + }); + + it('does not dial for a network that is not joined', () => { + const { networks, mock } = seeded([ADDR_A]); + (networks as any).joinedNetworks = new Set(); + edit(networks, [ADDR_B]); + expect(mock.dialledLists).toEqual([]); + expect(mock.prunedStatus).toHaveLength(1); + }); +}); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 6aaaf31f2..872386397 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -467,3 +467,54 @@ describe('addBootstrapPeers — superseded bootstrap configuration', () => { expect(dialled).toEqual([ADDR_A, ADDR_B]); }); }); + +/** + * A dial already handed to libp2p cannot be recalled: hangUp closes connections that + * already exist, so a leave-network landing mid-dial finds nothing to close and the + * connection appears a moment after the cleanup finished. Abandoning the loop is not + * enough — the connection has to be closed too. + */ +describe('addBootstrapPeers — a dial that lands after leave-network', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(suppressed: string[]) { + const disconnected: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).disconnectPeer = async (peerID: string): Promise => { + disconnected.push(peerID); + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + // The leave happens while this dial is in flight. + (network as any).redialSuppressedByNet.get('net-a').add(PEER_ID); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, disconnected }; + } + + it('closes a connection that arrived after the peer was left', async () => { + const { network, disconnected } = bareNetwork([]); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(disconnected).toEqual([PEER_ID]); + }); + + it('leaves an ordinary dial connected', async () => { + const { network, disconnected } = bareNetwork([]); + // No leave lands this time: the dial does not add the peer to the suppression set. + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(disconnected).toEqual([]); + }); +}); From 2c55adc137acb61cfc727628f90b2080b8d2eb09 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 18:33:58 +0200 Subject: [PATCH 41/99] fix(network): drop a deleted bootstrap peer from the autodial list --- backend/src/protocol/network.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 7d7140518..09f6cb76c 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1567,6 +1567,14 @@ export class Network { */ pruneConfiguredBootstrapPeer(peerID: string): void { this.configuredBootstrapPeerIDs.delete(peerID); + // Forget its addresses too. They were pushed into the autodial list when the + // entry was first configured, and that list is what zero-connection recovery + // walks — leaving them there means a bootstrap the user has just deleted keeps + // being dialed whenever the node runs out of connections, which is exactly the + // churn this work removes. The dedup set has to let go as well, or a later + // re-add would be treated as already known and the address could never come back. + this.bootstrapPeerIDs.delete(peerID); + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => extractDestinationPeerID(ma) !== peerID); } isBootstrapOrRelayPeer(peerID: string): boolean { From f9fc984aebb28ed677166ff4fb9e2f5e00af2491 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 18:33:58 +0200 Subject: [PATCH 42/99] fix(lishnets): store the cleaned bootstrap list, not the raw form rows --- backend/src/lishnet/lishnets.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 604ac6e6d..9886d3430 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -389,14 +389,17 @@ export class Networks { update(network: LISHNetworkConfig): boolean { const existing = this.get(network.networkID); - const ok = updateLISHnet(this.db, network); + // Store the cleaned list, not the raw one: blank rows from the form would + // otherwise be persisted while the runtime worked from the filtered copy, and + // the two would disagree about what this network's bootstrap list even is. + const cleaned = Networks.cleanBootstrapList(network.bootstrapPeers ?? []); + const ok = updateLISHnet(this.db, { ...network, bootstrapPeers: cleaned }); // The general edit form carries the bootstrap list as well, so this path can // change it just like updateBootstrapPeers does. Without the same runtime // synchronisation the edit would reach only the database and the live node // would keep dialing the previous list until restart. if (!ok || !existing) return ok; const previous = Networks.cleanBootstrapList(existing.bootstrapPeers); - const cleaned = Networks.cleanBootstrapList(network.bootstrapPeers ?? []); if (previous.join('\n') !== cleaned.join('\n')) this.syncBootstrapRuntime(network.networkID, existing.bootstrapPeers, cleaned); return ok; } From 5793ee338c672319b1a58553c8e21dd5dbb34b20 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 18:33:58 +0200 Subject: [PATCH 43/99] test(network): cover autodial cleanup and cleaned list persistence --- .../tests/unit/lishnet/leave-network.test.ts | 15 ++++++++-- .../protocol/peer-eviction-guards.test.ts | 28 +++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 9a9133967..f92ef2c91 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, beforeEach } from 'bun:test'; import { Database } from 'bun:sqlite'; -import { initLISHnetsTables, addLISHnet } from '../../../src/db/lishnets.ts'; +import { initLISHnetsTables, addLISHnet, getLISHnet } from '../../../src/db/lishnets.ts'; import { Networks } from '../../../src/lishnet/lishnets.ts'; /** @@ -316,7 +316,7 @@ describe('Networks.update — a changed bootstrap list reaches the running node' (networks as any).network = mock; (networks as any).db = db; (networks as any).joinedNetworks = new Set([NET]); - return { networks, mock }; + return { networks, mock, db }; } const edit = (networks: Networks, bootstrapPeers: string[]): boolean => (networks as any).update({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled: true, created: '2026-01-01T00:00:00.000Z' }); @@ -342,6 +342,17 @@ describe('Networks.update — a changed bootstrap list reaches the running node' expect(mock.prunedBootstrap).toEqual([]); }); + /** + * The form can submit blank rows. Persisting them raw while the runtime worked from + * the filtered copy left the database and the live node disagreeing about what the + * network's bootstrap list actually is. + */ + it('persists the cleaned list, not the blank rows the form submitted', () => { + const { networks, db } = seeded([ADDR_A]); + edit(networks, ['', ADDR_B, ' ']); + expect(getLISHnet(db, NET)?.bootstrapPeers).toEqual([ADDR_B]); + }); + it('does not dial for a network that is not joined', () => { const { networks, mock } = seeded([ADDR_A]); (networks as any).joinedNetworks = new Set(); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 872386397..923970e06 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from 'bun:test'; +import { multiaddr } from '@multiformats/multiaddr'; import { Network, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; /** @@ -368,6 +369,8 @@ describe('configured exemption ends when the peer leaves the config', () => { (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - 45 * 60_000]]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]; (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; @@ -401,6 +404,31 @@ describe('configured exemption ends when the peer leaves the config', () => { network.pruneConfiguredBootstrapPeer(PEER_ID); expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(false); }); + /** + * The autodial list is what zero-connection recovery walks. A bootstrap the user + * has deleted must leave it too, or the node keeps dialing that address every time + * it runs out of connections — the churn this work is supposed to end. + */ + it('forgets the deleted bootstrap address, so recovery stops dialing it', () => { + const { network } = bareNetwork(); + expect((network as any).bootstrapMultiaddrs).toHaveLength(1); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + }); + + it('also forgets it in the dedup set, so a later re-add can restore the address', () => { + const { network } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); + + it('leaves the addresses of other peers alone', () => { + const other = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const { network } = bareNetwork(); + (network as any).bootstrapMultiaddrs.push(multiaddr(`/ip4/203.0.113.10/tcp/9090/p2p/${other}`)); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([`/ip4/203.0.113.10/tcp/9090/p2p/${other}`]); + }); }); /** From da56b069026c54a0e14622b015f66a91bca1a9c5 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 19:52:50 +0200 Subject: [PATCH 44/99] fix(network): let only a verified discovered address join the autodial list --- backend/src/protocol/network.ts | 48 +++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 09f6cb76c..e2a94a157 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1317,11 +1317,17 @@ export class Network { this.unreachableQuarantine.delete(peerID); } } - const alreadyKnown = !!peerID && this.bootstrapPeerIDs.has(peerID); - if (peerID && !alreadyKnown) { - this.bootstrapPeerIDs.add(peerID); - this.bootstrapMultiaddrs.push(ma); - } + // The identity set is the dedup that stops every gossip mention of the same + // peer from costing another dial, so it is claimed up front either way. + if (peerID) this.bootstrapPeerIDs.add(peerID); + // The autodial list is a different promise: zero-connection recovery walks + // it and dials everything on it. A CONFIGURED address belongs there at once + // — it is user data and recovery must keep trying it precisely while it is + // down. A DISCOVERED address is only a claim some peer made, so it earns + // its place by answering; it is added after a verified dial, below. Adding + // it here left every unreachable address a gossip flood could invent on the + // list for good, since an ordinary timeout has nothing that takes it off. + if (origin === 'configured') this.rememberBootstrapAddress(ma); console.debug('Adding bootstrap peer:', peer); this.bootstrapTracker.markPending(networkID, peer, peerID, origin); try { @@ -1382,6 +1388,10 @@ export class Network { trace(`[NET] bootstrap addr unverified (connection came back on another address), left pending: ${peer}`); continue; } + // A gossip-learned address has now answered on the endpoint it claimed, so + // it has earned its place in the autodial list. Unverified ones never get + // there, which is what keeps a flood of invented addresses off it. + if (origin === 'discovered' && verifiedThisAddr) this.rememberBootstrapAddress(ma); this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { @@ -1565,6 +1575,34 @@ export class Network { * Both callers already establish that the peer is configured in NO joined * network before calling, so this needs no refcount of its own. */ + /** + * Put an address on the autodial list that zero-connection recovery walks, unless + * it is already there. + * + * Membership is decided by the ADDRESS, not by the peer ID behind it: a bootstrap + * whose host or port the user edited keeps its identity, and an identity-keyed + * check would treat the new address as already known and never add it — leaving + * recovery dialing the address that was replaced. + */ + private rememberBootstrapAddress(ma: any): void { + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (this.bootstrapMultiaddrs.some(m => normalizeMultiaddrForCompare(m.toString()) === canonical)) return; + this.bootstrapMultiaddrs.push(ma); + } + + /** + * Take specific addresses off the autodial list. Used when a network's configured + * list changes: an entry that is gone must stop being dialed, and that includes + * the case where the peer ID stays and only its address moved, which + * {@link pruneConfiguredBootstrapPeer} cannot see because the identity is still + * configured. + */ + pruneBootstrapAddresses(addresses: string[]): void { + if (addresses.length === 0) return; + const drop = new Set(addresses.map(a => normalizeMultiaddrForCompare(a))); + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => !drop.has(normalizeMultiaddrForCompare(ma.toString()))); + } + pruneConfiguredBootstrapPeer(peerID: string): void { this.configuredBootstrapPeerIDs.delete(peerID); // Forget its addresses too. They were pushed into the autodial list when the From 85a21b031b11c1857d0b4e6ae1ee91e48d7eae4d Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 19:52:50 +0200 Subject: [PATCH 45/99] fix(lishnets): drop a replaced bootstrap address from the autodial list --- backend/src/lishnet/lishnets.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 9886d3430..0476b7599 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -185,6 +185,16 @@ export class Networks { } /** Configured-bootstrap peer IDs of every joined network except `exceptID`. */ + /** Bootstrap ADDRESSES configured for every joined network except `exceptID`. */ + private configuredBootstrapAddressesElsewhere(exceptID: string): Set { + const out = new Set(); + for (const nid of this.joinedNetworks) { + if (nid === exceptID) continue; + for (const address of Networks.cleanBootstrapList(this.get(nid)?.bootstrapPeers ?? [])) out.add(address); + } + return out; + } + private configuredBootstrapPeerIDsElsewhere(exceptID: string): Set { const out = new Set(); for (const nid of this.joinedNetworks) { @@ -479,6 +489,13 @@ export class Networks { for (const pid of Networks.bootstrapPeerIDsOf(previousPeers)) { if (!nextIDs.has(pid) && !elsewhere.has(pid)) this.network.pruneConfiguredBootstrapPeer(pid); } + // Addresses that left the list while their peer ID stayed — the user edited a + // host or port. The identity-level prune above cannot see those, so recovery + // would go on dialing the address that was replaced. + const keptAddresses = new Set(cleaned); + const elsewhereAddresses = this.configuredBootstrapAddressesElsewhere(id); + const dropped = Networks.cleanBootstrapList(previousPeers).filter(a => !keptAddresses.has(a) && !elsewhereAddresses.has(a)); + this.network.pruneBootstrapAddresses(dropped); this.network.pruneBootstrapStatus(id, cleaned); if (this.joinedNetworks.has(id) && cleaned.length > 0) { this.network.addBootstrapPeers(cleaned, id, 'configured').catch(err => { From cd6c993a9592322c05994afc70be134bd512051b Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 19:52:51 +0200 Subject: [PATCH 46/99] test(network): cover autodial promotion and replaced bootstrap address --- .../tests/unit/lishnet/leave-network.test.ts | 27 +++++++ .../protocol/peer-eviction-guards.test.ts | 81 +++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index f92ef2c91..064805e86 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -28,6 +28,8 @@ interface MockNet { pruneConfiguredBootstrapPeer(pid: string): void; bumpBootstrapGeneration(networkID: string): void; generationBumps: string[]; + pruneBootstrapAddresses(addresses: string[]): void; + prunedAddresses: string[][]; pruneBootstrapStatus(networkID: string, keep: string[]): void; prunedStatus: Array<{ networkID: string; keep: string[] }>; addBootstrapPeers(peers: string[], networkID: string, origin: string): Promise; @@ -47,6 +49,7 @@ function makeMockNet(): MockNet { prunedBootstrap: [], suppressionClearedFor: [], generationBumps: [], + prunedAddresses: [], prunedStatus: [], dialledLists: [], getTopicPeers(id) { @@ -75,6 +78,9 @@ function makeMockNet(): MockNet { bumpBootstrapGeneration(networkID) { this.generationBumps.push(networkID); }, + pruneBootstrapAddresses(addresses) { + this.prunedAddresses.push(addresses); + }, pruneBootstrapStatus(networkID, keep) { this.prunedStatus.push({ networkID, keep }); }, @@ -353,6 +359,27 @@ describe('Networks.update — a changed bootstrap list reaches the running node' expect(getLISHnet(db, NET)?.bootstrapPeers).toEqual([ADDR_B]); }); + /** + * Editing only the host or port keeps the peer ID, so the identity-level prune sees + * nothing to do. Without an address-level prune the replaced address stays on the + * autodial list and recovery keeps dialing it. + */ + it('drops the replaced address when only the host changed', () => { + const moved = `/ip4/203.0.113.99/tcp/9090/p2p/${PEER_A}`; + const { networks, mock } = seeded([ADDR_A]); + edit(networks, [moved]); + expect(mock.prunedAddresses).toEqual([[ADDR_A]]); + expect(mock.prunedBootstrap).toEqual([]); + }); + + it('keeps an address that is still configured for another joined network', () => { + const { networks, mock } = seeded([ADDR_A]); + (networks as any).get = (nid: string) => (nid === 'net-other' ? { networkID: nid, bootstrapPeers: [ADDR_A] } : { networkID: NET, bootstrapPeers: [ADDR_A] }); + (networks as any).joinedNetworks = new Set([NET, 'net-other']); + edit(networks, [ADDR_B]); + expect(mock.prunedAddresses).toEqual([[]]); + }); + it('does not dial for a network that is not joined', () => { const { networks, mock } = seeded([ADDR_A]); (networks as any).joinedNetworks = new Set(); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 923970e06..a2bb6dc84 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -546,3 +546,84 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { expect(disconnected).toEqual([]); }); }); + +/** + * The autodial list is what zero-connection recovery dials. A gossip mention is only + * a claim, so an address that never answered must not end up on it — otherwise a peer + * naming many unreachable addresses parks them all there permanently, since an + * ordinary timeout has no cleanup path (only identity-mismatch does). + */ +describe('addBootstrapPeers — only a working discovered address joins the autodial list', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { fail?: boolean; remoteAddr?: string }) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (opts.fail) throw new Error('dial timeout'); + return { remoteAddr: { toString: () => opts.remoteAddr ?? ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('keeps a failed discovered address off the list', async () => { + const network = bareNetwork({ fail: true }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([]); + }); + + it('adds a discovered address that answered on the endpoint it claimed', async () => { + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([ADDR]); + }); + + it('keeps a discovered address off the list when the connection came back on another one', async () => { + const network = bareNetwork({ remoteAddr: `/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}` }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([]); + }); + + /** + * A configured entry is user data: recovery has to keep trying it precisely while + * it is down, so it joins the list before the dial and stays even when that fails. + */ + it('keeps a failed configured address on the list', async () => { + const network = bareNetwork({ fail: true }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR]); + }); + + it('does not add the same address twice', async () => { + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR]); + }); + + /** + * The dedup used to be keyed on the peer id, so a bootstrap whose host the user + * edited was treated as already known and its new address never reached the list. + */ + it('adds a new address of a peer it already knows', async () => { + const moved = `/ip4/203.0.113.99/tcp/9090/p2p/${PEER_ID}`; + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + await (network as any).addBootstrapPeers([moved], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR, moved]); + }); +}); From eac24f762770584f8fa932e4f2990490dce75203 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 20:15:33 +0200 Subject: [PATCH 47/99] fix(network): keep configured status for a non-routable bootstrap address --- backend/src/protocol/network.ts | 52 ++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index e2a94a157..d7ed0aeec 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1279,17 +1279,15 @@ export class Network { const superseded = (): boolean => epoch !== this.runEpoch || generation !== this.bootstrapGenerationOf(networkID); for (const peer of peers) { if (superseded()) return; + let probeAfterQuarantine = false; try { const ma = Multiaddr(peer); - // Safety net: refuse to add loopback / unreachable-private bootstrap - // entries even if the upstream (catalog or peer-announce intake) - // failed to filter them. Failing here is silent because the call - // site iterates many candidates and we shouldn't spam INFO for - // every drop; trace-level keeps it greppable when debugging. - if (shouldDenyDial(ma, localCidrs)) { - trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); - continue; - } + // Claim the configured status BEFORE the routability filter. Whether an address + // is dialable is a property of THIS HOST right now — a LAN or VPN bootstrap stops + // passing the filter the moment that interface drops — while "the user configured + // this peer" is a fact about the saved config. Deriving the second from the first + // left a VPN bootstrap unregistered whenever the tunnel was down at startup, so the + // exemption that makes configured peers un-evictable never applied to it. const peerID = extractDestinationPeerID(ma); // Skip our own address — compare the DESTINATION identity, not the raw // string: `/p2p//p2p-circuit/p2p/` contains our ID as the @@ -1303,6 +1301,16 @@ export class Network { // explicit dial fails or the connection drops before the next tick. this.clearRedialSuppressionForPeer(peerID); } + // Safety net: refuse to dial loopback / unreachable-private bootstrap entries + // even if the upstream (catalog or peer-announce intake) failed to filter them. + // A discovered address is dropped silently — the call site iterates many + // candidates and should not spam INFO for each. A configured one gets a status + // row instead: the user wrote it down and needs to see why nothing happens with it. + if (shouldDenyDial(ma, localCidrs)) { + trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); + if (origin === 'configured') this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'error', 'address is not routable from this host', null, origin); + continue; + } // Skip peers recently evicted as unreachable — nodes that still remember // them keep gossiping their addrs, and without this window every mention // would re-create the status row and burn a dial. Configured entries are @@ -1315,6 +1323,11 @@ export class Network { continue; } this.unreachableQuarantine.delete(peerID); + // This dial is the ONE probe an expired quarantine buys. If it fails the + // window has to close again — otherwise every later gossip mention spends + // another dial and refreshes the status row, which is exactly the churn + // the quarantine exists to stop. + probeAfterQuarantine = true; } } // The identity set is the dedup that stops every gossip mention of the same @@ -1363,7 +1376,11 @@ export class Network { // nothing to close and this connection surfaces a moment after the cleanup // finished. Abandoning the loop would leave it open, so close it here — the // suppression set is what says the user deliberately left this peer. - if (peerID && networkID && this.isRedialSuppressed(peerID)) { + // Two ways this connection can already be unwanted: the peer was hung up + // by leave-network and sits in the suppression set, or we left the network + // this dial belonged to before ever seeing the peer as one of its members, + // in which case it never entered that set at all. + if (peerID && networkID && (this.isRedialSuppressed(peerID) || !this.isTopicSubscribed(networkID))) { trace(`[NET] bootstrap dial landed after leave, disconnecting: ${peerID.slice(0, 16)}`); await this.disconnectPeer(peerID, networkID); return; @@ -1398,6 +1415,9 @@ export class Network { if (superseded()) return; const message = err?.message ?? String(err); const kind = classifyBootstrapError(message); + // The probe the expired quarantine allowed has failed, so close the window + // again rather than letting the next announce buy another dial. + if (probeAfterQuarantine && peerID) this.unreachableQuarantine.set(peerID, Date.now()); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, origin); // [NET-MISMATCH] richer log for identity-mismatch — single line containing @@ -1920,6 +1940,18 @@ export class Network { if (handlers) handlers.delete(handler); } + /** + * Whether this node is still subscribed to a lishnet's topic, i.e. has not left it. + * + * Without pubsub there is no answer to give, and the caller uses this to decide + * whether to tear a connection down — so the unknown case reports "still joined", + * which is the harmless one. + */ + isTopicSubscribed(networkID: string): boolean { + if (!this.pubsub) return true; + return this.pubsub.getTopics().includes(lishTopic(networkID)); + } + unsubscribeTopic(networkID: string): void { if (!this.pubsub) return; const topic = lishTopic(networkID); From 29a21c9e8271d117ac06f07817bfa0917ced01d7 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 20:15:33 +0200 Subject: [PATCH 48/99] fix(lishnets): compare configured bootstrap addresses canonically --- backend/src/lishnet/lishnets.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 0476b7599..fa31bfff0 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -1,5 +1,5 @@ import { type Database } from 'bun:sqlite'; -import { Network } from '../protocol/network.ts'; +import { Network, normalizeMultiaddrForCompare } from '../protocol/network.ts'; import { Utils } from '../utils.ts'; import { type DataServer } from '../lish/data-server.ts'; import { type Settings } from '../settings.ts'; @@ -185,12 +185,12 @@ export class Networks { } /** Configured-bootstrap peer IDs of every joined network except `exceptID`. */ - /** Bootstrap ADDRESSES configured for every joined network except `exceptID`. */ + /** Canonical bootstrap ADDRESSES configured for every joined network except `exceptID`. */ private configuredBootstrapAddressesElsewhere(exceptID: string): Set { const out = new Set(); for (const nid of this.joinedNetworks) { if (nid === exceptID) continue; - for (const address of Networks.cleanBootstrapList(this.get(nid)?.bootstrapPeers ?? [])) out.add(address); + for (const address of Networks.cleanBootstrapList(this.get(nid)?.bootstrapPeers ?? [])) out.add(normalizeMultiaddrForCompare(address)); } return out; } @@ -492,9 +492,12 @@ export class Networks { // Addresses that left the list while their peer ID stayed — the user edited a // host or port. The identity-level prune above cannot see those, so recovery // would go on dialing the address that was replaced. - const keptAddresses = new Set(cleaned); + // Compare canonically, the same way the autodial list itself does. Raw string + // equality would treat two spellings of one address (DNS case, IPv6 form) as + // different entries here and as the same one during the prune below. + const keptAddresses = new Set(cleaned.map(normalizeMultiaddrForCompare)); const elsewhereAddresses = this.configuredBootstrapAddressesElsewhere(id); - const dropped = Networks.cleanBootstrapList(previousPeers).filter(a => !keptAddresses.has(a) && !elsewhereAddresses.has(a)); + const dropped = Networks.cleanBootstrapList(previousPeers).filter(a => !keptAddresses.has(normalizeMultiaddrForCompare(a)) && !elsewhereAddresses.has(normalizeMultiaddrForCompare(a))); this.network.pruneBootstrapAddresses(dropped); this.network.pruneBootstrapStatus(id, cleaned); if (this.joinedNetworks.has(id) && cleaned.length > 0) { From 420d3b8a5c055dd8b689f65393a01868f3fca135 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 20:15:33 +0200 Subject: [PATCH 49/99] test(network): cover non-routable configured entry, quarantine and leave --- .../tests/unit/lishnet/leave-network.test.ts | 13 ++ .../protocol/peer-eviction-guards.test.ts | 141 ++++++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 064805e86..c74dbd5f6 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -380,6 +380,19 @@ describe('Networks.update — a changed bootstrap list reaches the running node' expect(mock.prunedAddresses).toEqual([[]]); }); + /** + * The autodial prune compares addresses canonically, so the "did this entry leave + * the list" check has to as well — otherwise one spelling of an address counts as a + * removal here and as the same entry there. + */ + it('treats two spellings of one address as the same entry', () => { + const upper = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PEER_A}`; + const lower = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PEER_A}`; + const { networks, mock } = seeded([upper]); + edit(networks, [lower]); + expect(mock.prunedAddresses).toEqual([[]]); + }); + it('does not dial for a network that is not joined', () => { const { networks, mock } = seeded([ADDR_A]); (networks as any).joinedNetworks = new Set(); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index a2bb6dc84..898012f57 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -538,6 +538,27 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { expect(disconnected).toEqual([PEER_ID]); }); + /** + * The peer we were dialing may never have been seen as a member of the network, so + * leave-network had nothing to hang up and never put it in the suppression set. + * Leaving the topic is the fact that decides it, not the suppression bookkeeping. + */ + it('closes a connection to a peer we never saw as a member of the network we left', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + (network as any).pubsub = { getTopics: () => [] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([PEER_ID]); + }); + + it('keeps the connection while the network is still joined', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + (network as any).pubsub = { getTopics: () => ['lish/net-a'] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + it('leaves an ordinary dial connected', async () => { const { network, disconnected } = bareNetwork([]); // No leave lands this time: the dial does not add the peer to the suppression set. @@ -627,3 +648,123 @@ describe('addBootstrapPeers — only a working discovered address joins the auto expect(addresses(network)).toEqual([ADDR, moved]); }); }); + +/** + * Whether an address is dialable is a fact about THIS HOST right now — a LAN or VPN + * bootstrap stops passing the routability filter the moment that interface drops. + * Whether the user configured a peer is a fact about the saved config. Deriving the + * second from the first left a VPN bootstrap unregistered whenever the tunnel was + * down at startup, and an unregistered configured peer loses the exemption that is + * supposed to make it un-evictable. + */ +describe('addBootstrapPeers — a non-routable configured entry is still configured', () => { + // A private address in a subnet this host is not on: the filter rejects it. + const OFF_VPN = `/ip4/10.201.0.5/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const outcomes: Array<{ status: string; message: string | null }> = []; + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { + markPending() {}, + recordOutcome(_n: unknown, _a: unknown, _p: unknown, status: string, message: string | null) { + outcomes.push({ status, message }); + }, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, outcomes, dialled }; + } + + it('registers the peer as configured even though the address is not routable', async () => { + const { network } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + it('still does not dial an address the filter rejected', async () => { + const { network, dialled } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect(dialled).toEqual([]); + }); + + it('tells the user why the configured entry is doing nothing', async () => { + const { network, outcomes } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect(outcomes).toEqual([{ status: 'error', message: 'address is not routable from this host' }]); + }); + + it('says nothing about a discovered address the filter rejected', async () => { + const { network, outcomes } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'discovered'); + expect(outcomes).toEqual([]); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); +}); + +/** + * An expired quarantine buys exactly one probe. Letting a failed probe pass without + * closing the window again means every later gossip mention spends another dial and + * refreshes the status row — the churn the quarantine exists to stop. + */ +describe('addBootstrapPeers — quarantine after the probe it allowed', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(fail: boolean, quarantinedAt: number) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map([[PEER_ID, quarantinedAt]]); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (fail) throw new Error('dial timeout'); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const LONG_AGO = Date.now() - 10 * 60 * 60_000; + + it('re-arms the quarantine when the allowed probe fails', async () => { + const network = bareNetwork(true, LONG_AGO); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + const at = (network as any).unreachableQuarantine.get(PEER_ID); + expect(at).toBeGreaterThan(LONG_AGO); + }); + + it('leaves the quarantine lifted when the probe succeeds', async () => { + const network = bareNetwork(false, LONG_AGO); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); + + it('does not quarantine a plain failure that was never in one', async () => { + const network = bareNetwork(true, LONG_AGO); + (network as any).unreachableQuarantine = new Map(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); +}); From be3ad48ca76fa5fbf243ea81143353567035f3ca Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 20:54:30 +0200 Subject: [PATCH 50/99] fix(network): keep a peer another joined lishnet still needs --- backend/src/protocol/network.ts | 71 ++++++++++++++++++++++++++++----- 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index d7ed0aeec..dd50b1c58 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -130,6 +130,21 @@ const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; * would evict the whole non-configured peerStore on the first dial after the * connection came back. */ +/** + * Whether zero-connection recovery may dial a DISCOVERED address this tick. + * + * It answers with the two records re-dial maintenance already keeps: a peer inside its + * backoff window waits for it to expire, and one still inside its unreachable + * quarantine stays down. Both are delays, never permanent bans — the point is only + * that the recovery loop must not undo the pacing the other loop just applied. + */ +export function isRecoveryDialDue(peerID: string, now: number, redialBackoff: ReadonlyMap, quarantine: ReadonlyMap): boolean { + const quarantinedAt = quarantine.get(peerID); + if (quarantinedAt !== undefined && now - quarantinedAt < UNREACHABLE_QUARANTINE_MS) return false; + const backoff = redialBackoff.get(peerID); + return backoff === undefined || backoff.nextAttempt <= now; +} + export function nextEvictionWindowStart(reachable: boolean, previous: number | undefined, now: number): number { return reachable ? (previous ?? now) : now; } @@ -1133,6 +1148,18 @@ export class Network { const p2pComponents = ma.getComponents().filter((c: { code: number; value?: string }) => c.code === 421); const pid: string | undefined = p2pComponents.length > 0 ? p2pComponents[p2pComponents.length - 1].value : undefined; if (pid && this.isRedialSuppressed(pid)) continue; // deliberately left — don't resurrect it here + // A CONFIGURED entry is the user's way back in and is always tried; a + // DISCOVERED one earned its place here by answering once, but that is no + // reason to bypass the pacing re-dial maintenance applies to it. Without + // this, an isolated node re-dialed a dead discovered peer every 30s + // forever, since maintenance stops counting failures the moment we have no + // other connection to prove we are online. + const configured = !!pid && this.configuredBootstrapPeerIDs.has(pid); + if (pid && !configured && !isRecoveryDialDue(pid, Date.now(), this.redialBackoff, this.unreachableQuarantine)) continue; + // Routability is re-checked here, not just at configure time: a LAN or VPN + // bootstrap is on this list while its interface is down, and becomes dialable + // again the moment it returns. + if (shouldDenyDial(ma, getLocalCidrs())) continue; const maStr = ma?.toString?.() ?? String(ma); // Each dial awaits for up to 10s, so a stop() can land mid-loop; the // remaining dials belong to a node this run no longer owns. @@ -1301,6 +1328,18 @@ export class Network { // explicit dial fails or the connection drops before the next tick. this.clearRedialSuppressionForPeer(peerID); } + // Also before the routability filter: a LAN or VPN bootstrap is unroutable only + // while its interface is down, and keeping it off the recovery list until then + // means nothing retries it when the tunnel returns. Recovery re-checks + // routability itself before dialing. + // The autodial list is a different promise: zero-connection recovery walks + // it and dials everything on it. A CONFIGURED address belongs there at once + // — it is user data and recovery must keep trying it precisely while it is + // down. A DISCOVERED address is only a claim some peer made, so it earns + // its place by answering; it is added after a verified dial, below. Adding + // it here left every unreachable address a gossip flood could invent on the + // list for good, since an ordinary timeout has nothing that takes it off. + if (origin === 'configured') this.rememberBootstrapAddress(ma); // Safety net: refuse to dial loopback / unreachable-private bootstrap entries // even if the upstream (catalog or peer-announce intake) failed to filter them. // A discovered address is dropped silently — the call site iterates many @@ -1333,14 +1372,6 @@ export class Network { // The identity set is the dedup that stops every gossip mention of the same // peer from costing another dial, so it is claimed up front either way. if (peerID) this.bootstrapPeerIDs.add(peerID); - // The autodial list is a different promise: zero-connection recovery walks - // it and dials everything on it. A CONFIGURED address belongs there at once - // — it is user data and recovery must keep trying it precisely while it is - // down. A DISCOVERED address is only a claim some peer made, so it earns - // its place by answering; it is added after a verified dial, below. Adding - // it here left every unreachable address a gossip flood could invent on the - // list for good, since an ordinary timeout has nothing that takes it off. - if (origin === 'configured') this.rememberBootstrapAddress(ma); console.debug('Adding bootstrap peer:', peer); this.bootstrapTracker.markPending(networkID, peer, peerID, origin); try { @@ -1380,7 +1411,7 @@ export class Network { // by leave-network and sits in the suppression set, or we left the network // this dial belonged to before ever seeing the peer as one of its members, // in which case it never entered that set at all. - if (peerID && networkID && (this.isRedialSuppressed(peerID) || !this.isTopicSubscribed(networkID))) { + if (peerID && networkID && (this.isRedialSuppressed(peerID) || !this.isTopicSubscribed(networkID)) && !this.isPeerNeededByJoinedNetwork(peerID)) { trace(`[NET] bootstrap dial landed after leave, disconnecting: ${peerID.slice(0, 16)}`); await this.disconnectPeer(peerID, networkID); return; @@ -1683,6 +1714,28 @@ export class Network { * LISHs just because a transport connection exists (e.g. the peer's * keep-alive re-dialed us right after we left its network). */ + /** + * Whether a lishnet we are STILL in has any claim on this peer. + * + * Leaving one network says nothing about the others — the same peer can be a member + * of a second lishnet or its configured bootstrap. Tearing it down is destructive + * (disconnectPeer suppresses re-dials AND drops the peerStore entry), so that is + * reserved for a peer no joined network has a use for. It is the same question + * leaveNetwork asks before it hangs anyone up. + */ + private isPeerNeededByJoinedNetwork(peerID: string): boolean { + if (this.isBootstrapOrRelayPeer(peerID)) return true; + if (this.sharesJoinedTopicWith(peerID)) return true; + if (!this.pubsub) return false; + for (const topic of this.pubsub.getTopics()) { + if (!topic.startsWith(LISH_TOPIC_PREFIX)) continue; + // Recently-seen counts as well: a member that is momentarily disconnected is + // still a member, and leaveNetwork widens its own snapshot the same way. + if (this.peerAnnounce.getRecentMembers(topic).includes(peerID)) return true; + } + return false; + } + sharesJoinedTopicWith(peerID: string): boolean { if (!this.pubsub) return false; for (const topic of this.pubsub.getTopics()) { From 5d4fb47204404f6cdd6bf1b0c7bdc0d749da564b Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 20:54:30 +0200 Subject: [PATCH 51/99] test(network): cover shared-peer leave, recovery pacing and parked address --- .../unit/protocol/network-disconnect.test.ts | 34 +++++++++++ .../protocol/peer-eviction-guards.test.ts | 60 ++++++++++++++++++- 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index d2fc16f27..f90cbf7d4 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -180,6 +180,9 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const dialed: string[] = []; const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = bootstrapMaStrs.map(s => multiaddr(s)); (network as any).recentDisconnects = []; (network as any).bootstrapTracker = { entries: () => [] }; @@ -200,6 +203,37 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { expect(dialed).toEqual([]); }); + /** + * Recovery shares the pacing records with re-dial maintenance. Without that an + * isolated node re-dialed a dead discovered peer every tick forever: maintenance + * stops counting its failures once there is no other connection to prove we are + * online, so nothing else was slowing it down. + */ + it('skips a discovered bootstrap peer inside its backoff window', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); + await run(network, []); + expect(dialed).toEqual([]); + }); + + it('still dials a CONFIGURED peer inside a backoff window — it is the way back in', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + await run(network, []); + expect(dialed).toEqual([multiaddr(ma).toString()]); + }); + + it('skips a discovered bootstrap peer still inside its unreachable quarantine', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).unreachableQuarantine = new Map([[PEER_ID, Date.now() - 60_000]]); + await run(network, []); + expect(dialed).toEqual([]); + }); + it('still dials a non-suppressed bootstrap peer', async () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([], [ma]); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 898012f57..b1c2b5020 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { multiaddr } from '@multiformats/multiaddr'; -import { Network, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; +import { Network, isRecoveryDialDue, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; /** * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered @@ -534,10 +534,30 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { it('closes a connection that arrived after the peer was left', async () => { const { network, disconnected } = bareNetwork([]); - await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); expect(disconnected).toEqual([PEER_ID]); }); + /** + * Tearing the peer down is destructive — disconnectPeer suppresses re-dials and + * drops the peerStore entry — so leaving ONE network must not do it to a peer + * another joined network still has a claim on. Here the peer is configured + * infrastructure, which is claim enough. + */ + it('keeps a connection to a peer another joined network still needs', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + + it('keeps a connection to a peer that still subscribes another joined topic', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).pubsub = { getTopics: () => ['lish/net-b'], getSubscribers: () => [{ toString: () => PEER_ID }] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + /** * The peer we were dialing may never have been seen as a member of the network, so * leave-network had nothing to hang up and never put it in the suppression set. @@ -708,6 +728,16 @@ describe('addBootstrapPeers — a non-routable configured entry is still configu expect(outcomes).toEqual([{ status: 'error', message: 'address is not routable from this host' }]); }); + /** + * It has to be on the recovery list even while unroutable, or nothing retries it + * when the interface comes back. Recovery re-checks routability before dialing. + */ + it('still puts the unroutable configured address on the recovery list', async () => { + const { network } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([OFF_VPN]); + }); + it('says nothing about a discovered address the filter rejected', async () => { const { network, outcomes } = bareNetwork(); await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'discovered'); @@ -768,3 +798,29 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); }); }); + +/** + * Zero-connection recovery used to dial every address it held, ignoring the pacing + * re-dial maintenance had just applied. On an isolated node that meant a dead + * discovered peer was re-dialed every tick forever, because maintenance stops + * counting failures the moment there is no other connection to prove we are online. + */ +describe('isRecoveryDialDue', () => { + const now = 1_000_000; + const none = new Map(); + const noBackoff = new Map(); + + it('allows a peer with no backoff and no quarantine', () => { + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, none)).toBe(true); + }); + + it('holds a peer back inside its backoff window and releases it after', () => { + expect(isRecoveryDialDue(PEER_ID, now, new Map([[PEER_ID, { nextAttempt: now + 1 }]]), none)).toBe(false); + expect(isRecoveryDialDue(PEER_ID, now, new Map([[PEER_ID, { nextAttempt: now }]]), none)).toBe(true); + }); + + it('holds a quarantined peer back until the window passes', () => { + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, new Map([[PEER_ID, now - 60_000]]))).toBe(false); + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, new Map([[PEER_ID, now - 10 * 60 * 60_000]]))).toBe(true); + }); +}); From 97155e2fe0e1bbe7d2c22db0c07380632e50c4c7 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 21:22:41 +0200 Subject: [PATCH 52/99] fix(network): trim the disproved address instead of the whole peer --- backend/src/protocol/network.ts | 115 +++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 23 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index dd50b1c58..464ca2fa7 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1175,12 +1175,71 @@ export class Network { } } + /** + * Slowly re-probe CONFIGURED bootstrap addresses that nothing else will reach. + * + * An address the routability filter rejected at configure time — a LAN or VPN + * bootstrap whose interface was down — never entered the peerStore, so re-dial + * maintenance (which walks the peerStore) has no candidate for it. Zero-connection + * recovery would pick it up, but only while the node has NO connections at all, so a + * node happily talking to someone else would never notice the tunnel came back. + * + * Runs on the slow promote cadence and honours the same pacing as every other loop, + * so a permanently broken entry costs one dial per window, not one per tick. + */ + private async probeParkedConfiguredBootstraps(epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; + const localCidrs = getLocalCidrs(); + const now = Date.now(); + for (const ma of [...this.bootstrapMultiaddrs]) { + if (epoch !== this.runEpoch) return; + const pid = extractDestinationPeerID(ma); + if (!pid || !this.configuredBootstrapPeerIDs.has(pid)) continue; + if (this.isRedialSuppressed(pid)) continue; + // Still unreachable from here — leave it parked for a later pass. + if (shouldDenyDial(ma, localCidrs)) continue; + if (!isRecoveryDialDue(pid, now, this.redialBackoff, this.unreachableQuarantine)) continue; + try { + if (node.getConnections(peerIDFromString(pid)).length > 0) continue; + } catch { + continue; // unparseable id — nothing sane to probe + } + try { + await node.dial(ma, { signal: AbortSignal.timeout(10000) }); + if (epoch !== this.runEpoch) return; + this.redialBackoff.delete(pid); + console.log(`[NET] parked configured bootstrap reachable again: ${ma.toString()}`); + } catch (err: any) { + if (epoch !== this.runEpoch) return; + const previous = this.redialBackoff.get(pid); + const failCount = previous?.failCount ?? 0; + // Paces itself and nothing more: a configured peer is exempt from eviction, + // so this probe must never become the evidence that evicts one. + this.redialBackoff.set(pid, { + nextAttempt: Date.now() + Math.min(30_000 * 2 ** failCount, 600_000), + failCount: failCount + 1, + firstFailure: previous?.firstFailure ?? Date.now(), + evictionFails: previous?.evictionFails ?? 0, + }); + trace(`[NET] parked configured bootstrap still failing: ${ma.toString()} — ${err?.message ?? err}`); + } + } + } + private async maybePromotePeers(epoch: number = this.runEpoch): Promise { // Every 5th status tick (~150 s at 30 s status cadence) promote every // CONNECTED peer back to bootstrap priority (KEEP_ALIVE re-stamp + gossipsub // direct set). Disconnected peers are handled by runRedialMaintenance. this.statusTickCount++; if (this.statusTickCount % 5 === 0) { + try { + // Same slow cadence: an address parked as unroutable has no other loop + // that would ever notice its interface came back. + await this.probeParkedConfiguredBootstraps(epoch); + } catch (err: any) { + trace(`[NET] probeParkedConfiguredBootstraps failed: ${err?.message ?? err}`); + } try { await this.promoteKnownPeersToBootstrap(epoch); } catch (err: any) { @@ -1469,30 +1528,40 @@ export class Network { // address; peer with no connections → full purge as before. if (kind === 'identity-mismatch' && peerID) { const pid = peerIDFromString(peerID); - if (this.node.getConnections(pid).length > 0) { - // Compare in a form that survives both multiaddr normalization - // (expanded → compressed IPv6) and DNS case / trailing-dot - // differences — otherwise a filter that fails to match silently - // keeps the poisoned address while logging that it was dropped. - const canonical = normalizeMultiaddrForCompare(ma.toString()); - const canonicalBare = canonical.replace(/\/p2p\/[^/]+$/, ''); - const matches = (s: string): boolean => { - const n = normalizeMultiaddrForCompare(s); - return n === canonical || n === canonicalBare; - }; - // Restrict the autodial-list filter to entries of THIS peer so a - // case-insensitive compare can never drop a different peer's addr. - this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => extractDestinationPeerID(m) !== peerID || !matches(m.toString())); - try { - const rec = await this.node.peerStore.get(pid); - const keep = rec.addresses.filter((a: any) => !matches(a.multiaddr.toString())); - if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); - } catch { - /* peer not in store — nothing to trim */ - } - console.log(`[NET] dropped stale addr of connected peer ${peerID.slice(0, 16)}: ${ma.toString()}`); + // Compare in a form that survives both multiaddr normalization (expanded → + // compressed IPv6) and DNS case / trailing-dot differences — otherwise a + // filter that fails to match silently keeps the poisoned address while + // logging that it was dropped. + const canonical = normalizeMultiaddrForCompare(ma.toString()); + const canonicalBare = canonical.replace(/\/p2p\/[^/]+$/, ''); + const matches = (str: string): boolean => { + const n = normalizeMultiaddrForCompare(str); + return n === canonical || n === canonicalBare; + }; + // Noise proves exactly one thing: THIS address no longer leads to the peer + // we expected. It says nothing about the peer's other addresses, so the bad + // one goes first and unconditionally — whether or not the peer happens to be + // connected right this moment. Purging on "not currently connected" threw + // away addresses that were never disproved. + // Restrict the autodial-list filter to entries of THIS peer so a + // case-insensitive compare can never drop a different peer's addr. + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => extractDestinationPeerID(m) !== peerID || !matches(m.toString())); + let remainingAddresses = 0; + try { + const rec = await this.node.peerStore.get(pid); + const keep = rec.addresses.filter((a: any) => !matches(a.multiaddr.toString())); + if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); + remainingAddresses = keep.length; + } catch { + /* peer not in store — nothing to trim, and nothing left either */ + } + if (superseded()) return; + // Only once the peer has neither a live connection nor a single address we + // have not disproved is there anything left to purge. + if (this.node.getConnections(pid).length === 0 && remainingAddresses === 0) { + await this.purgeStalePeer(peerID, `${origin} dial identity mismatch, no usable address left`, epoch); } else { - await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`, epoch); + console.log(`[NET] dropped stale addr of peer ${peerID.slice(0, 16)}: ${ma.toString()}`); } // For DISCOVERED entries (peer-announce gossip), also drop the // status entry — there's no saved config row to "fix" and leaving From 8c0c1905ae169adea9ddaf6a116cbb82bf972a83 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 21:22:41 +0200 Subject: [PATCH 53/99] test(network): cover identity-mismatch trim and parked bootstrap probe --- .../protocol/peer-eviction-guards.test.ts | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index b1c2b5020..5eb15e74f 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -824,3 +824,129 @@ describe('isRecoveryDialDue', () => { expect(isRecoveryDialDue(PEER_ID, now, noBackoff, new Map([[PEER_ID, now - 10 * 60 * 60_000]]))).toBe(true); }); }); + +/** + * Noise proves one thing only: THIS address no longer leads to the peer we expected. + * Purging the whole peer because it happened to be disconnected threw away addresses + * that were never disproved — a peer reachable tomorrow on its other address was + * forgotten because one stale entry answered with the wrong identity today. + */ +describe('addBootstrapPeers — identity mismatch trims the address, not the peer', () => { + const BAD = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const GOOD = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(storedAddresses: string[]) { + const purged: string[] = []; + const patched: string[][] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).purgeStalePeer = async (id: string): Promise => { + purged.push(id); + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(): Promise { + throw new Error(`Payload identity key 12D3KooWSomeoneElseSomeoneElseSomeoneElseSomeoneEls does not match expected remote identity key ${PEER_ID}`); + }, + peerStore: { + async get(): Promise { + return { addresses: storedAddresses.map(a => ({ multiaddr: multiaddr(a) })) }; + }, + async patch(_pid: unknown, data: { multiaddrs: Array<{ toString(): string }> }): Promise { + patched.push(data.multiaddrs.map(m => m.toString())); + }, + async merge(): Promise {}, + }, + }; + return { network, purged, patched }; + } + + it('keeps a disconnected peer that still has an undisproved address', async () => { + const { network, purged, patched } = bareNetwork([BAD, GOOD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect(purged).toEqual([]); + expect(patched).toEqual([[GOOD]]); + }); + + it('purges only once nothing usable is left', async () => { + const { network, purged } = bareNetwork([BAD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect(purged).toEqual([PEER_ID]); + }); + + it('drops the disproved address from the autodial list either way', async () => { + const { network } = bareNetwork([BAD, GOOD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + }); +}); + +/** + * An address the routability filter rejected never reaches the peerStore, so re-dial + * maintenance has no candidate for it, and zero-connection recovery only runs with NO + * connections at all. Without this slow pass a node talking to someone else would + * never notice a VPN bootstrap became reachable again. + */ +describe('probeParkedConfiguredBootstraps', () => { + const PARKED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { configured?: boolean; connections?: number } = {}) { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured === false ? [] : [PEER_ID]); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapMultiaddrs = [multiaddr(PARKED)]; + (network as any).node = { + getConnections: () => Array.from({ length: opts.connections ?? 0 }, () => ({})), + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + }, + }; + return { network, dialed }; + } + + const run = (network: Network): Promise => (network as any).probeParkedConfiguredBootstraps(1); + + it('probes a configured address even though we hold other connections', async () => { + const { network, dialed } = bareNetwork(); + await run(network); + expect(dialed).toEqual([multiaddr(PARKED).toString()]); + }); + + it('leaves a discovered address to the loops that own it', async () => { + const { network, dialed } = bareNetwork({ configured: false }); + await run(network); + expect(dialed).toEqual([]); + }); + + it('does not re-probe a peer that is already connected', async () => { + const { network, dialed } = bareNetwork({ connections: 1 }); + await run(network); + expect(dialed).toEqual([]); + }); + + it('respects the backoff so a broken entry costs one dial per window', async () => { + const { network, dialed } = bareNetwork(); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + await run(network); + expect(dialed).toEqual([]); + }); + + it('skips a peer the user left', async () => { + const { network, dialed } = bareNetwork(); + (network as any).redialSuppressedByNet = new Map([['net-a', new Set([PEER_ID])]]); + await run(network); + expect(dialed).toEqual([]); + }); +}); From 5fffaf3098a6198e719cca65547e774a3424a0e2 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 21:40:23 +0200 Subject: [PATCH 54/99] fix(network): track configured bootstrap origin per address, not per peer --- backend/src/protocol/network.ts | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 464ca2fa7..86ad598ae 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -241,6 +241,16 @@ export class Network { * until restart. */ private configuredBootstrapPeerIDs: Set = new Set(); + /** + * Canonical bootstrap ADDRESSES that came from saved config, as opposed to gossip. + * + * Kept alongside the peer-ID set because the two answer different questions. Whether + * a PEER may be auto-evicted is about identity — configured anywhere means exempt. + * Whether an ADDRESS gets the configured treatment in recovery is about that address: + * one peer can have a configured address and a gossip-learned one at the same time, + * and the gossip-learned one must not inherit the exemption from its sibling. + */ + private readonly configuredBootstrapAddresses: Set = new Set(); private dcutrPeers: Set = new Set(); private bootstrapMultiaddrs: any[] = []; /** @@ -1154,7 +1164,7 @@ export class Network { // this, an isolated node re-dialed a dead discovered peer every 30s // forever, since maintenance stops counting failures the moment we have no // other connection to prove we are online. - const configured = !!pid && this.configuredBootstrapPeerIDs.has(pid); + const configured = this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(ma.toString())); if (pid && !configured && !isRecoveryDialDue(pid, Date.now(), this.redialBackoff, this.unreachableQuarantine)) continue; // Routability is re-checked here, not just at configure time: a LAN or VPN // bootstrap is on this list while its interface is down, and becomes dialable @@ -1195,7 +1205,7 @@ export class Network { for (const ma of [...this.bootstrapMultiaddrs]) { if (epoch !== this.runEpoch) return; const pid = extractDestinationPeerID(ma); - if (!pid || !this.configuredBootstrapPeerIDs.has(pid)) continue; + if (!pid || !this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(ma.toString()))) continue; if (this.isRedialSuppressed(pid)) continue; // Still unreachable from here — leave it parked for a later pass. if (shouldDenyDial(ma, localCidrs)) continue; @@ -1398,7 +1408,10 @@ export class Network { // its place by answering; it is added after a verified dial, below. Adding // it here left every unreachable address a gossip flood could invent on the // list for good, since an ordinary timeout has nothing that takes it off. - if (origin === 'configured') this.rememberBootstrapAddress(ma); + if (origin === 'configured') { + this.configuredBootstrapAddresses.add(normalizeMultiaddrForCompare(ma.toString())); + this.rememberBootstrapAddress(ma); + } // Safety net: refuse to dial loopback / unreachable-private bootstrap entries // even if the upstream (catalog or peer-announce intake) failed to filter them. // A discovered address is dropped silently — the call site iterates many @@ -1721,6 +1734,7 @@ export class Network { if (addresses.length === 0) return; const drop = new Set(addresses.map(a => normalizeMultiaddrForCompare(a))); this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => !drop.has(normalizeMultiaddrForCompare(ma.toString()))); + for (const address of drop) this.configuredBootstrapAddresses.delete(address); } pruneConfiguredBootstrapPeer(peerID: string): void { @@ -1732,7 +1746,17 @@ export class Network { // churn this work removes. The dedup set has to let go as well, or a later // re-add would be treated as already known and the address could never come back. this.bootstrapPeerIDs.delete(peerID); - this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => extractDestinationPeerID(ma) !== peerID); + // Only the addresses that came from the config. The same peer may also have a + // gossip-learned address that earned its place by answering a dial — that one + // belongs to the discovered lifecycle (TTL, backoff) and is not the user's to lose + // just because they deleted a different address of the same peer. + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => { + if (extractDestinationPeerID(ma) !== peerID) return true; + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (!this.configuredBootstrapAddresses.has(canonical)) return true; + this.configuredBootstrapAddresses.delete(canonical); + return false; + }); } isBootstrapOrRelayPeer(peerID: string): boolean { @@ -2369,6 +2393,7 @@ export class Network { this.unreachableQuarantine.clear(); this.noReachableSince.clear(); this.configuredBootstrapPeerIDs.clear(); + this.configuredBootstrapAddresses.clear(); this.redialSuppressedByNet.clear(); this.pxIngressLogKeys.clear(); if (this.node) { From 5079e643ab6100d55e16d96d9a047c15b3f6b7a7 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 21:40:24 +0200 Subject: [PATCH 55/99] test(network): cover per-address configured origin --- .../unit/protocol/network-disconnect.test.ts | 6 +- .../protocol/peer-eviction-guards.test.ts | 74 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index f90cbf7d4..c80c2629e 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'bun:test'; import { KEEP_ALIVE } from '@libp2p/interface'; import { multiaddr } from '@multiformats/multiaddr'; -import { Network } from '../../../src/protocol/network.ts'; +import { Network, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; /** * Unit tests for Network.disconnectPeer tag hygiene: hanging up a peer must @@ -127,6 +127,7 @@ describe('Network.runRedialMaintenance — leave-peer suppression', () => { (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); // A reconnected peer's suppression is lifted only if it currently shares a joined // topic — model that via a pubsub whose subscribers list the "back on topic" peers. (network as any).pubsub = { @@ -183,6 +184,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { (network as any).redialBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapMultiaddrs = bootstrapMaStrs.map(s => multiaddr(s)); (network as any).recentDisconnects = []; (network as any).bootstrapTracker = { entries: () => [] }; @@ -222,6 +224,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const { network, dialed } = bareNetwork([], [ma]); (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(multiaddr(ma).toString())]); await run(network, []); expect(dialed).toEqual([multiaddr(ma).toString()]); }); @@ -252,6 +255,7 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapGeneration = new Map(); (network as any).bootstrapPeerIDs = new Set(); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 5eb15e74f..61e56b4fc 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -94,6 +94,7 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; @@ -159,6 +160,7 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -321,6 +323,7 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -369,8 +372,10 @@ describe('configured exemption ends when the peer leaves the config', () => { (network as any).unreachableQuarantine = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - 45 * 60_000]]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapPeerIDs = new Set([PEER_ID]); (network as any).bootstrapMultiaddrs = [multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]; + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]); (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; @@ -450,6 +455,7 @@ describe('addBootstrapPeers — superseded bootstrap configuration', () => { (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -511,6 +517,7 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -547,6 +554,7 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { it('keeps a connection to a peer another joined network still needs', async () => { const { network, disconnected } = bareNetwork([]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(); await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); expect(disconnected).toEqual([]); }); @@ -602,6 +610,7 @@ describe('addBootstrapPeers — only a working discovered address joins the auto (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -688,6 +697,7 @@ describe('addBootstrapPeers — a non-routable configured entry is still configu (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -759,6 +769,7 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map([[PEER_ID, quarantinedAt]]); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; @@ -842,6 +853,7 @@ describe('addBootstrapPeers — identity mismatch trims the address, not the pee (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; @@ -904,6 +916,7 @@ describe('probeParkedConfiguredBootstraps', () => { (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured === false ? [] : [PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(opts.configured === false ? [] : [normalizeMultiaddrForCompare(PARKED)]); (network as any).unreachableQuarantine = new Map(); (network as any).redialBackoff = new Map(); (network as any).bootstrapMultiaddrs = [multiaddr(PARKED)]; @@ -950,3 +963,64 @@ describe('probeParkedConfiguredBootstraps', () => { expect(dialed).toEqual([]); }); }); + +/** + * One peer can hold a configured address AND a gossip-learned one at the same time. + * Deciding "is this configured" from the peer id let the gossip-learned sibling + * inherit the configured exemption — no backoff, no quarantine, and it survived the + * user deleting the configured entry it had nothing to do with. + */ +describe('configured origin is a property of the address, not the peer', () => { + const CONFIGURED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const DISCOVERED = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(CONFIGURED)]); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED), multiaddr(DISCOVERED)]; + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).node = { + getPeers: () => [], + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + }, + }; + return { network, dialed }; + } + + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('paces a discovered address whose peer is configured under a DIFFERENT address', async () => { + // Only the discovered address is on the list, so nothing else can satisfy the + // loop: if it gets dialed while inside its backoff window, it was wrongly given + // the configured exemption its sibling address owns. + const { network, dialed } = bareNetwork(); + (network as any).bootstrapMultiaddrs = [multiaddr(DISCOVERED)]; + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + await (network as any).runZeroConnectionRecovery([]); + expect(dialed).toEqual([]); + }); + + it('still lets the configured address through the same backoff', async () => { + const { network, dialed } = bareNetwork(); + (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED)]; + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + await (network as any).runZeroConnectionRecovery([]); + expect(dialed).toEqual([multiaddr(CONFIGURED).toString()]); + }); + + it('keeps the discovered address when the configured one is deleted', () => { + const { network } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect(addresses(network)).toEqual([multiaddr(DISCOVERED).toString()]); + }); +}); From c0dcd147386d0432bf21b6553f25370f58ebdbaa Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:01:04 +0200 Subject: [PATCH 56/99] refactor(protocol): share one destination peer id and canonical address helper --- backend/src/protocol/multiaddr-utils.ts | 85 +++++++++++++++++++++++++ backend/src/protocol/network-config.ts | 8 +-- backend/src/protocol/network.ts | 69 ++++++++++---------- 3 files changed, 126 insertions(+), 36 deletions(-) create mode 100644 backend/src/protocol/multiaddr-utils.ts diff --git a/backend/src/protocol/multiaddr-utils.ts b/backend/src/protocol/multiaddr-utils.ts new file mode 100644 index 000000000..8c41155ab --- /dev/null +++ b/backend/src/protocol/multiaddr-utils.ts @@ -0,0 +1,85 @@ +/** + * Multiaddr helpers shared by the runtime network layer and the libp2p config + * builder. + * + * They live in their own module because `network-config.ts` runs before, and is + * imported by, `network.ts` — pulling the whole network module in just to reach a + * two-line helper would make that cycle. Everything here is pure. + */ + +import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; + +/** Multiaddr component code of a `/p2p/` segment. */ +export const MULTIADDR_P2P_CODE = 421; + +/** + * Peer ID we would actually connect to when dialing a multiaddr, or null when the + * address carries no `/p2p` component. + * + * A relayed address (`.../p2p//p2p-circuit/p2p/`) carries two `/p2p` + * components; the destination — the peer whose identity the Noise handshake verifies + * — is the LAST one. Taking the first returns the relay instead, which then gets + * protected, tagged or suppressed in place of the peer actually meant. + */ +export function extractDestinationPeerID(ma: any): string | null { + try { + const components = ma.getComponents().filter((c: { code: number }) => c.code === MULTIADDR_P2P_CODE); + return components.length > 0 ? (components[components.length - 1].value ?? null) : null; + } catch { + return null; + } +} + +/** Same as {@link extractDestinationPeerID} but taking the address as a string. */ +export function destinationPeerIDOf(address: string): string | null { + try { + return extractDestinationPeerID(Multiaddr(address.trim())); + } catch { + return null; + } +} + +/** + * Bounded memo for the canonical form. Canonicalising parses the address, and callers + * compare one address against the whole bootstrap list, so the same handful of strings + * would be re-parsed constantly. The cap stops a flood of gossip-invented addresses + * from turning the cache into a leak — it is only a cache, so clearing it wholesale + * costs a recompute and nothing else. + */ +const CANONICAL_CACHE_LIMIT = 4096; +const canonicalCache = new Map(); + +/** + * One spelling per address, for comparing two multiaddrs that mean the same thing. + * + * The string goes through the multiaddr parser first, which is what collapses an + * expanded IPv6 literal to its compressed form and settles component ordering — a regex + * over the raw text cannot do that, so two spellings of one address would count as two + * different entries. Only then is the DNS host folded: DNS is case-insensitive and may + * carry the FQDN root dot, while a base58 peer ID in the same string is + * case-SIGNIFICANT and must survive untouched. + * + * An unparseable input comes back trimmed rather than throwing: callers use this for + * equality between values they already hold, and "compares equal only to itself" is the + * safe answer there. + */ +export function canonicalMultiaddr(address: string): string { + const cached = canonicalCache.get(address); + if (cached !== undefined) return cached; + const result = computeCanonicalMultiaddr(address); + if (canonicalCache.size >= CANONICAL_CACHE_LIMIT) canonicalCache.clear(); + canonicalCache.set(address, result); + return result; +} + +/** Uncached canonicalisation — see {@link canonicalMultiaddr}. */ +function computeCanonicalMultiaddr(address: string): string { + const trimmed = address.trim(); + let parsed = trimmed; + try { + parsed = Multiaddr(trimmed).toString(); + } catch { + // keep the trimmed original + } + return parsed.replace(/\/(dns|dns4|dns6|dnsaddr)\/([^/]+)/gi, (_match, protocol: string, host: string) => `/${protocol.toLowerCase()}/${host.toLowerCase().replace(/\.+$/, '')}`); +} diff --git a/backend/src/protocol/network-config.ts b/backend/src/protocol/network-config.ts index 555940ce3..84f8c6c08 100644 --- a/backend/src/protocol/network-config.ts +++ b/backend/src/protocol/network-config.ts @@ -25,6 +25,7 @@ import { trace } from '../logger.ts'; import { normalizeTrustedPeerIds, parseAcceptPXThreshold } from './constants.ts'; import { getLocalCidrs, shouldDenyDial, extractFirstIPv4 } from './address-filter.ts'; import { peerIdFromString } from '@libp2p/peer-id'; +import { extractDestinationPeerID } from './multiaddr-utils.ts'; const { multiaddr: Multiaddr } = await import('@multiformats/multiaddr'); /** A gossipsub direct-peer entry: a peer id and its multiaddrs. */ @@ -45,7 +46,7 @@ function buildDirectPeersFromBootstrap(uniquePeers: string[]): DirectPeer[] { for (const ma of uniquePeers) { try { const parsed = Multiaddr(ma); - const pid = parsed.getComponents().find((c: any) => c.code === 421)?.value; + const pid = extractDestinationPeerID(parsed); if (!pid) continue; direct.push({ id: peerIdFromString(pid), addrs: [parsed] }); } catch { @@ -210,8 +211,7 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult // trusted peers blocked when their advertised addr lives on a LAN segment // different from our own. Trusted peers are by policy known-good // destinations, so dial them regardless of CIDR match. - const pidComponent = ma?.getComponents?.()?.find?.((c: any) => c.code === 421); - const pid = pidComponent?.value ?? null; + const pid = extractDestinationPeerID(ma); if (pid && (bootstrapPeerIDs.has(pid) || trustedPXPeerIDs.has(pid))) return false; const deny = shouldDenyDial(ma, getLocalCidrs()); if (deny) { @@ -410,7 +410,7 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult console.log(' -', peer); try { const ma = Multiaddr(peer); - const peerID = ma.getComponents().find(c => c.code === 421)?.value ?? null; + const peerID = extractDestinationPeerID(ma); if (peerID) { bootstrapPeerIDs.add(peerID); bootstrapMultiaddrs.push(ma); diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 86ad598ae..4bbdab94d 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -15,6 +15,7 @@ import { buildLibp2pConfig } from './network-config.ts'; import { type WantMessage } from './downloader.ts'; import { lishTopic, LISH_TOPIC_PREFIX } from './constants.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; +import { canonicalMultiaddr, extractDestinationPeerID } from './multiaddr-utils.ts'; import { CodedError, ErrorCodes, type NetworkNodeInfo, type PeerConnectionInfo, type IMeshHealth, type BootstrapStatus, type BootstrapPeerDialStatus, type BootstrapPeerOrigin } from '@shared'; import { Circuit } from '@multiformats/multiaddr-matcher'; import { createTopicScoreParams } from '@chainsafe/libp2p-gossipsub/score'; @@ -201,6 +202,13 @@ export class Network { private statusInterval: NodeJS.Timeout | null = null; /** Monotonic counter for status-interval ticks. Used by the periodic autodial promotion. */ private statusTickCount = 0; + /** + * Delayed peer-count probes armed by subscribeTopic. Tracked so stop() can cancel + * them: without that they keep a closure on this instance alive and can fire against + * a node the run no longer owns, which is exactly the ownership the epoch guards + * elsewhere are there to enforce. + */ + private readonly delayedPeerCountTimers: Set> = new Set(); /** Guards against overlapping status ticks — see setupStatusInterval. */ private statusTickInFlight = false; /** @@ -454,6 +462,17 @@ export class Network { /** * Schedule a debounced check of peer counts for all subscribed topics. */ + /** Arm a one-shot peer-count probe that stop() can still cancel. */ + private armDelayedPeerCountCheck(delayMs: number): void { + const epoch = this.runEpoch; + const timer = setTimeout(() => { + this.delayedPeerCountTimers.delete(timer); + if (epoch !== this.runEpoch) return; + this.schedulePeerCountCheck(); + }, delayMs); + this.delayedPeerCountTimers.add(timer); + } + private schedulePeerCountCheck(): void { if (this._peerCountDebounceTimer) clearTimeout(this._peerCountDebounceTimer); this._peerCountDebounceTimer = setTimeout(() => { @@ -2073,9 +2092,7 @@ export class Network { this.topicHandlers.get(topic)!.add(handler); console.log(`✓ Subscribed to lishnet topic: ${topic}`); // GossipSub mesh needs time to rebuild after subscribe — schedule delayed peer count checks - setTimeout(() => this.schedulePeerCountCheck(), 2000); - setTimeout(() => this.schedulePeerCountCheck(), 5000); - setTimeout(() => this.schedulePeerCountCheck(), 15000); + for (const delay of [2000, 5000, 15000]) this.armDelayedPeerCountCheck(delay); } /** @@ -2394,6 +2411,11 @@ export class Network { this.noReachableSince.clear(); this.configuredBootstrapPeerIDs.clear(); this.configuredBootstrapAddresses.clear(); + // Per-run like the rest of this state: a fresh node must not inherit a count that + // makes its very first tick the slow-cadence one. + this.statusTickCount = 0; + for (const timer of this.delayedPeerCountTimers) clearTimeout(timer); + this.delayedPeerCountTimers.clear(); this.redialSuppressedByNet.clear(); this.pxIngressLogKeys.clear(); if (this.node) { @@ -2430,19 +2452,22 @@ export class Network { } } +// Re-exported so callers and tests that already reach for this here keep working; the +// implementation lives in multiaddr-utils so network-config can share it without +// importing this module. +export { extractDestinationPeerID }; + /** - * Normalize a multiaddr STRING for equality comparison. Multiaddr.toString() - * already compresses IPv6, but leaves DNS host case and the FQDN root dot intact — - * `/dns4/EXAMPLE.COM./tcp/...` and `/dns4/example.com/tcp/...` address the same - * endpoint. + * Normalize a multiaddr STRING for equality comparison. * - * Only the HOST of a DNS component is folded, never the whole address. A circuit - * multiaddr carries `/p2p/` in the middle, and a base58 peer ID is - * case-significant — case-folding an identifier is a different question from - * case-folding a hostname, and this function is only entitled to the second. + * Delegates to {@link canonicalMultiaddr}, which parses the address first — that is + * what folds an expanded IPv6 literal into its compressed form. Doing it with a regex + * over the raw text, as this used to, left `/ip6/2001:0db8:0000:...:0001` and + * `/ip6/2001:db8::1` looking like two different addresses even though they are one, + * so a configuration edit could leave the old spelling behind in the autodial list. */ export function normalizeMultiaddrForCompare(s: string): string { - return s.replace(/\/(dns|dns4|dns6|dnsaddr)\/([^/]+)/gi, (_match, protocol: string, host: string) => `/${protocol.toLowerCase()}/${host.toLowerCase().replace(/\.+$/, '')}`); + return canonicalMultiaddr(s); } /** @@ -2460,26 +2485,6 @@ export function isSameDialEndpoint(a: string, b: string): boolean { return left.length > 0 && left === strip(b); } -/** - * Extract the DESTINATION peer ID from a multiaddr. A circuit-relay address has - * the shape `/.../p2p//p2p-circuit/p2p/` — taking the FIRST - * /p2p/ component would return the relay's identity, so eviction and configured - * protection would target the wrong peer. The last /p2p/ component is always - * the dial target. Returns null when the multiaddr carries no peer ID at all. - */ -export function extractDestinationPeerID(ma: any): string | null { - try { - const components: Array<{ code: number; value?: string }> = ma?.getComponents?.() ?? []; - for (let i = components.length - 1; i >= 0; i--) { - const c = components[i]!; - if (c.code === 421 && typeof c.value === 'string') return c.value; - } - } catch { - /* unparseable multiaddr — no ID */ - } - return null; -} - /** * Classify a libp2p dial error into a coarse status the UI can render distinctly. * From b24da950ea72cdac8c9a6019ea40f66c050b4010 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:01:04 +0200 Subject: [PATCH 57/99] fix(lishnets): trim and dedupe the bootstrap list, honour the write result --- backend/src/lishnet/lishnets.ts | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index fa31bfff0..5c015f0ca 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -1,5 +1,6 @@ import { type Database } from 'bun:sqlite'; import { Network, normalizeMultiaddrForCompare } from '../protocol/network.ts'; +import { canonicalMultiaddr } from '../protocol/multiaddr-utils.ts'; import { Utils } from '../utils.ts'; import { type DataServer } from '../lish/data-server.ts'; import { type Settings } from '../settings.ts'; @@ -460,14 +461,37 @@ export class Networks { if (!existing) return null; const cleaned = Networks.cleanBootstrapList(bootstrapPeers); const next: LISHNetworkConfig = { ...existing, bootstrapPeers: cleaned }; - updateLISHnet(this.db, next); + // Persist first and believe the answer. Switching the runtime over after a failed + // write would leave the node dialing a list the database never accepted, and the + // old one would come back at the next restart with nothing to explain the change. + if (!updateLISHnet(this.db, next)) throw new CodedError(ErrorCodes.NETWORK_NOT_FOUND, id); this.syncBootstrapRuntime(id, existing.bootstrapPeers, cleaned); return next; } - /** Drop blank entries from a user-supplied bootstrap list. */ + /** + * Normalise a user-supplied bootstrap list: drop blanks, trim, and keep one entry per + * canonical address. + * + * Trimming matters because the list came from a text field — a value with stray + * whitespace passed the old blank check and then failed to parse at dial time. + * Deduplicating matters because two spellings of one address (DNS case, expanded vs + * compressed IPv6) would otherwise each get their own forced probe and their own + * status row for the same endpoint. + */ private static cleanBootstrapList(peers: string[]): string[] { - return peers.filter(p => typeof p === 'string' && p.trim().length > 0); + const seen = new Set(); + const out: string[] = []; + for (const raw of peers) { + if (typeof raw !== 'string') continue; + const trimmed = raw.trim(); + if (trimmed.length === 0) continue; + const key = canonicalMultiaddr(trimmed); + if (seen.has(key)) continue; + seen.add(key); + out.push(trimmed); + } + return out; } /** From 998fa817570a502d1ed22140d9411b6c67709008 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:01:04 +0200 Subject: [PATCH 58/99] test(protocol): cover canonical addresses, relay target and per-run stop state --- .../unit/protocol/multiaddr-utils.test.ts | 81 +++++++++++++++++++ .../tests/unit/protocol/network-mesh.test.ts | 53 ++++++++++-- .../protocol/peer-eviction-guards.test.ts | 64 +++++++++++++++ 3 files changed, 191 insertions(+), 7 deletions(-) create mode 100644 backend/tests/unit/protocol/multiaddr-utils.test.ts diff --git a/backend/tests/unit/protocol/multiaddr-utils.test.ts b/backend/tests/unit/protocol/multiaddr-utils.test.ts new file mode 100644 index 000000000..74c5da8c0 --- /dev/null +++ b/backend/tests/unit/protocol/multiaddr-utils.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from 'bun:test'; +import { canonicalMultiaddr, destinationPeerIDOf, extractDestinationPeerID } from '../../../src/protocol/multiaddr-utils.ts'; +import { multiaddr } from '@multiformats/multiaddr'; + +const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const RELAY = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + +/** + * These helpers decide whether two addresses are "the same one", which downstream + * decides whether an entry is replaced, kept, or dialed again. Getting either half + * wrong is invisible until an address quietly lingers in the autodial list. + */ +describe('canonicalMultiaddr', () => { + /** + * The regex-only version this replaced could not do this: the two spellings are one + * address, and treating them as two left a replaced bootstrap behind after an edit. + */ + it('folds an expanded IPv6 literal to its compressed form', () => { + const expanded = `/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090/p2p/${PEER_A}`; + const compressed = `/ip6/2001:db8::1/tcp/9090/p2p/${PEER_A}`; + expect(canonicalMultiaddr(expanded)).toBe(canonicalMultiaddr(compressed)); + }); + + it('folds DNS host case and the FQDN root dot', () => { + expect(canonicalMultiaddr('/dns4/EXAMPLE.COM./tcp/443')).toBe('/dns4/example.com/tcp/443'); + }); + + it('leaves a peer id alone — base58 is case-significant', () => { + expect(canonicalMultiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`)).toContain(PEER_A); + }); + + it('trims surrounding whitespace, which a text field can easily carry in', () => { + expect(canonicalMultiaddr(` /ip4/203.0.113.9/tcp/9090/p2p/${PEER_A} `)).toBe(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`); + }); + + it('keeps genuinely different addresses different', () => { + expect(canonicalMultiaddr('/ip4/203.0.113.9/tcp/80')).not.toBe(canonicalMultiaddr('/ip4/203.0.113.9/tcp/8080')); + }); + + it('returns an unparseable value trimmed rather than throwing', () => { + expect(canonicalMultiaddr(' not-a-multiaddr ')).toBe('not-a-multiaddr'); + }); + + it('is stable when applied twice', () => { + const once = canonicalMultiaddr(`/ip6/2001:0db8::0001/tcp/9090/p2p/${PEER_A}`); + expect(canonicalMultiaddr(once)).toBe(once); + }); +}); + +describe('extractDestinationPeerID', () => { + it('returns the peer id of a plain address', () => { + expect(extractDestinationPeerID(multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`))).toBe(PEER_A); + }); + + /** + * The whole reason this helper exists: the first /p2p component of a circuit address + * is the relay, and protecting or tagging the relay in place of the peer we meant is + * how the wrong identity ends up in the bootstrap sets. + */ + it('returns the destination of a circuit address, not the relay', () => { + expect(extractDestinationPeerID(multiaddr(`/ip4/198.51.100.1/tcp/4001/p2p/${RELAY}/p2p-circuit/p2p/${PEER_A}`))).toBe(PEER_A); + }); + + it('returns null when the address carries no peer id', () => { + expect(extractDestinationPeerID(multiaddr('/ip4/203.0.113.9/tcp/9090'))).toBeNull(); + }); + + it('returns null instead of throwing on something that is not a multiaddr', () => { + expect(extractDestinationPeerID({} as unknown)).toBeNull(); + }); +}); + +describe('destinationPeerIDOf', () => { + it('accepts the address as a string, whitespace and all', () => { + expect(destinationPeerIDOf(` /ip4/203.0.113.9/tcp/9090/p2p/${PEER_A} `)).toBe(PEER_A); + }); + + it('returns null for an unparseable string', () => { + expect(destinationPeerIDOf('nonsense')).toBeNull(); + }); +}); diff --git a/backend/tests/unit/protocol/network-mesh.test.ts b/backend/tests/unit/protocol/network-mesh.test.ts index c6d46f8e1..f78de85e2 100644 --- a/backend/tests/unit/protocol/network-mesh.test.ts +++ b/backend/tests/unit/protocol/network-mesh.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'bun:test'; import { readFileSync } from 'fs'; import { join } from 'path'; import { LISH_TOPIC_PREFIX, DEFAULT_ACCEPT_PX_THRESHOLD, lishTopic, normalizeTrustedPeerIds, parseAcceptPXThreshold } from '../../../src/protocol/constants.ts'; +import { Network } from '../../../src/protocol/network.ts'; import { logStatusDebug } from '../../../src/protocol/status-logger.ts'; const NETWORK_TS = readFileSync(join(__dirname, '../../../src/protocol/network.ts'), 'utf-8'); @@ -348,20 +349,58 @@ describe('lishTopic helper', () => { // Peer count check scheduling — source code verification // --------------------------------------------------------------------------- -describe('subscribeTopic — peer count scheduling', () => { - it('schedules 3 delayed peer count checks after subscribe', () => { - const subscribeBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('subscribeTopic(networkID: string)'), NETWORK_TS.indexOf('unsubscribeHandler')); - const matches = subscribeBlock.match(/setTimeout\(\(\) => this\.schedulePeerCountCheck\(\)/g); - expect(matches).not.toBeNull(); - expect(matches!.length).toBe(3); +/** + * These used to grep the source for three literal setTimeout calls, which said nothing + * about what actually happens and broke the moment the calls were factored into a loop. + * The delayed probes matter for a different reason now: they used to be untracked, so + * they kept a closure on the instance alive and could fire against a node the run no + * longer owned. + */ +describe('subscribeTopic — delayed peer count probes', () => { + function bareNetwork() { + const checks: number[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).delayedPeerCountTimers = new Set(); + (network as any).schedulePeerCountCheck = () => checks.push(Date.now()); + return { network, checks }; + } + + it('tracks every probe it arms, so stop can cancel them', () => { + const { network } = bareNetwork(); + for (const delay of [2000, 5000, 15000]) (network as any).armDelayedPeerCountCheck(delay); + expect((network as any).delayedPeerCountTimers.size).toBe(3); + for (const timer of (network as any).delayedPeerCountTimers) clearTimeout(timer); }); - it('uses delays 2s, 5s, 15s for mesh rebuild', () => { + it('still uses the 2s, 5s and 15s mesh-rebuild delays', () => { const subscribeBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('subscribeTopic(networkID: string)'), NETWORK_TS.indexOf('unsubscribeHandler')); expect(subscribeBlock).toContain('2000'); expect(subscribeBlock).toContain('5000'); expect(subscribeBlock).toContain('15000'); }); + + it('does not check peer counts once the probe fires for a superseded run', async () => { + const { network, checks } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + (network as any).runEpoch = 2; // a stop()/start() landed while the probe was pending + await new Promise(resolve => setTimeout(resolve, 20)); + expect(checks).toEqual([]); + }); + + it('checks peer counts when the run is still the same', async () => { + const { network, checks } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(checks).toHaveLength(1); + }); + + it('forgets a timer once it has fired', async () => { + const { network } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + await new Promise(resolve => setTimeout(resolve, 20)); + expect((network as any).delayedPeerCountTimers.size).toBe(0); + }); }); describe('unsubscribeTopic — peer count scheduling', () => { diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 61e56b4fc..87624f156 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1024,3 +1024,67 @@ describe('configured origin is a property of the address, not the peer', () => { expect(addresses(network)).toEqual([multiaddr(DISCOVERED).toString()]); }); }); + +/** + * stop() clears the per-run state so a restart starts clean. The slow-cadence counter + * was left out, so a fresh node could inherit a count that made its very first tick the + * slow one — the opposite of the ownership the epoch guards enforce everywhere else. + * The delayed peer-count probes were likewise untracked and kept firing at a node the + * run no longer owned. + */ +describe('Network.stop — per-run state really is per run', () => { + function bareNetwork() { + const network = Object.create(Network.prototype) as Network; + for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys']) { + (network as any)[field] = field === 'seenSearchIDs' || field === 'dcutrPeers' || field === 'bootstrapPeerIDs' || field === 'configuredBootstrapPeerIDs' || field === 'configuredBootstrapAddresses' ? new Set() : new Map(); + } + (network as any).runEpoch = 1; + (network as any).statusInterval = null; + (network as any).wantResponseCleanupInterval = null; + (network as any)._peerCountDebounceTimer = null; + (network as any).listeners = []; + (network as any).bootstrapMultiaddrs = []; + (network as any).delayedPeerCountTimers = new Set(); + (network as any).peerAnnounce = { stop() {} }; + (network as any).bootstrapTracker = { clear() {} }; + (network as any).node = null; + (network as any).datastore = null; + return network; + } + + it('resets the slow-cadence tick counter', async () => { + const network = bareNetwork(); + (network as any).statusTickCount = 4; // one short of the slow tick + await network.stop(); + expect((network as any).statusTickCount).toBe(0); + }); + + /** + * The epoch guard already stops a late probe from DOING anything, so observing the + * callback proves nothing about cancellation. What has to be asserted is that the + * handle is actually released — otherwise a pending timer keeps a closure on the old + * instance alive until it fires. + */ + it('cancels the delayed peer-count probes it armed', async () => { + const network = bareNetwork(); + (network as any).armDelayedPeerCountCheck(60_000); + (network as any).armDelayedPeerCountCheck(60_000); + const armed = [...(network as any).delayedPeerCountTimers]; + expect(armed).toHaveLength(2); + + const realClearTimeout = globalThis.clearTimeout; + const cleared: unknown[] = []; + globalThis.clearTimeout = ((timer: unknown) => { + cleared.push(timer); + return realClearTimeout(timer as Parameters[0]); + }) as typeof globalThis.clearTimeout; + try { + await network.stop(); + } finally { + globalThis.clearTimeout = realClearTimeout; + } + + for (const timer of armed) expect(cleared).toContain(timer); + expect((network as any).delayedPeerCountTimers.size).toBe(0); + }); +}); From bd779da7e86e5c2d1c836b2402ea887d3293dad3 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:33:33 +0200 Subject: [PATCH 59/99] fix(protocol): count unique announced addresses against the intake cap --- backend/src/protocol/peer-announce.ts | 29 +++++-- .../tests/unit/protocol/peer-announce.test.ts | 75 +++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index 01d53078f..0f5469543 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -1,6 +1,7 @@ import { trace } from '../logger.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; +import { canonicalMultiaddr } from './multiaddr-utils.ts'; import { LISH_TOPIC_PREFIX } from './constants.ts'; import { type Libp2p } from 'libp2p'; import { type BootstrapPeerOrigin } from '@shared'; @@ -183,11 +184,19 @@ export class PeerAnnounceManager { // do the same — every receiver must be defensive. const localCidrs = getLocalCidrs(); const rawCount = data.multiaddrs.length; - const filtered: string[] = []; + // Third stage: collapse addresses that mean the same thing. The cap counts + // UNIQUE addresses, so a message repeating one address 128 times no longer + // consumes the whole budget — and, more to the point, no longer turns into + // 128 markPending + dial + recordOutcome rounds downstream, since every stage + // after this one keys off the address string. Canonicalisation (not raw string + // equality) is what makes two spellings of one address — DNS case, expanded vs + // compressed IPv6 — count once. + const unique = new Map(); let droppedNonRoutable = 0; + let droppedDuplicate = 0; for (const a of data.multiaddrs) { if (typeof a !== 'string' || a.length === 0) continue; - if (filtered.length >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; + if (unique.size >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; try { if (shouldDenyDial(Multiaddr(a), localCidrs)) { droppedNonRoutable++; @@ -198,13 +207,21 @@ export class PeerAnnounceManager { droppedNonRoutable++; continue; } - filtered.push(a); + const canonical = canonicalMultiaddr(a); + if (unique.has(canonical)) { + droppedDuplicate++; + continue; + } + // Keep the spelling as announced: downstream keys status rows by this exact + // string, and rewriting it here would split one peer's row in two. + unique.set(canonical, a); } - if (filtered.length === 0) { - if (droppedNonRoutable > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${droppedNonRoutable}/${rawCount} addrs as non-routable`); + if (unique.size === 0) { + if (droppedNonRoutable > 0 || droppedDuplicate > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${rawCount} addrs (${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate)`); return; } - trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, network ${networkID.slice(0, 8)})`); + const filtered = [...unique.values()]; + trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, network ${networkID.slice(0, 8)})`); // Pass networkID so per-peer outcomes from gossiped entries surface in the // UI under the network through which they arrived. Identity-mismatch // outcomes inside addBootstrapPeers also trigger purgeStalePeer. diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 03af65169..87c2ada9c 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -229,3 +229,78 @@ describe('PeerAnnounceManager.emit recently-seen membership', () => { } }); }); + +// Inbound-intake guards. Every address that survives handle() costs a dial, a status +// row and a snapshot downstream, so the two things that bound that cost — collapsing +// addresses that mean the same thing, and capping what one announcing peer can spend — +// are asserted here rather than left to the receivers further down the chain. + +const SRC_ID = '12D3KooWSourceSourceSourceSourceSourceSourceSourceSS'; + +/** A manager wired only for handle(): captures the address lists it forwards. */ +function intakeManager() { + const forwarded: string[][] = []; + const mgr = new PeerAnnounceManager({ + getNode: () => null, + getPubsub: () => null, + broadcast: async () => {}, + addBootstrapPeers: async multiaddrs => { + forwarded.push(multiaddrs); + }, + }); + return { mgr, forwarded }; +} + +/** N distinct routable addresses in RFC5737 TEST-NET-3. */ +function distinctAddrs(count: number): string[] { + return Array.from({ length: count }, (_v, i) => `/ip4/203.0.113.${i % 254}/tcp/${9000 + i}`); +} + +describe('PeerAnnounceManager.handle address dedup', () => { + it('collapses one address repeated many times into a single entry', async () => { + const { mgr, forwarded } = intakeManager(); + const addr = '/ip4/198.51.100.7/tcp/9090'; + + await mgr.handle({ type: 'peer-announce', multiaddrs: Array(300).fill(addr) }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[addr]]); + }); + + it('collapses two spellings of one address (DNS case, expanded vs compressed IPv6)', async () => { + const { mgr, forwarded } = intakeManager(); + const multiaddrs = [`/dns4/Peer.Example.COM/tcp/9090/p2p/${SELF_ID}`, `/dns4/peer.example.com/tcp/9090/p2p/${SELF_ID}`, '/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090', '/ip6/2001:db8::1/tcp/9090']; + + await mgr.handle({ type: 'peer-announce', multiaddrs }, 'netAAAA', SRC_ID); + + // One DNS entry + one IPv6 entry, each keeping the spelling it arrived in. + expect(forwarded[0]).toEqual([multiaddrs[0]!, multiaddrs[2]!]); + }); + + it('counts UNIQUE addresses against the total cap, not raw entries', async () => { + // 300 copies of one address plus 5 distinct ones is 6 unique — the duplicates + // must not consume the 128-address budget the distinct ones need. + const { mgr, forwarded } = intakeManager(); + const dup = '/ip4/198.51.100.7/tcp/9090'; + const rest = distinctAddrs(5); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [...Array(300).fill(dup), ...rest] }, 'netAAAA', SRC_ID); + + expect(forwarded[0]).toEqual([dup, ...rest]); + }); + + it('still caps a flood of genuinely distinct addresses at 128', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: distinctAddrs(200) }, 'netAAAA', SRC_ID); + + expect(forwarded[0]!.length).toBe(128); + }); + + it('drops non-routable addresses before deduping', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: ['/ip4/127.0.0.1/tcp/9090', '/ip4/127.0.0.1/tcp/9090', 'not-a-multiaddr'] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([]); + }); +}); From 91f39bd7d4a3d9a6c0262bdb991f1515d18b964b Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:34:23 +0200 Subject: [PATCH 60/99] feat(protocol): rate limit peer announce intake per source --- backend/src/protocol/peer-announce.ts | 89 ++++++++++++++++++- .../tests/unit/protocol/peer-announce.test.ts | 69 +++++++++++++- 2 files changed, 155 insertions(+), 3 deletions(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index 0f5469543..de0f6903f 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -61,6 +61,80 @@ const PEER_ANNOUNCE_MAX_ADDRS_PER_PEER = 3; */ const PEER_ANNOUNCE_MEMBER_TTL_MS = PEER_ANNOUNCE_INTERVAL_SATURATED_MS * 3; +/** + * Per-source announce budget, in unique addresses admitted per minute. + * + * Every address that survives intake costs a dial, a status row and a snapshot, and + * nothing about gossipsub stops one topic subscriber from announcing as fast as it + * likes — so the cost of a single hostile (or merely broken) emitter is otherwise + * unbounded. The budget is spent per announcing peer ID, so throttling one source + * never starves the rest of the topic. + * + * Sizing: the worst LEGITIMATE emitter is a mid-convergence peer (peerStore 20..80, + * {@link PEER_ANNOUNCE_INTERVAL_STEADY_MS} cadence) sending the full + * {@link PEER_ANNOUNCE_MAX_TOTAL_ADDRS} list twice a minute — 256 addresses. The + * sustained rate matches that exactly, and the bucket holds one and a half cycles' + * worth so downward jitter, a topic re-join or a cold-start burst still pass intact. + * A source that exceeds it is not a shape we emit. + */ +const PEER_ANNOUNCE_RATE_PER_MINUTE = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 2; +/** Bucket depth — burst allowance on top of {@link PEER_ANNOUNCE_RATE_PER_MINUTE}. */ +const PEER_ANNOUNCE_RATE_BURST = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 3; +/** + * Cap on tracked sources. Buckets are keyed by peer ID, which is unbounded input, so + * the map is an LRU: the least recently heard-from source is evicted first. Eviction + * hands that source a fresh budget if it ever returns, which is acceptable — refilling + * the table costs an attacker a distinct peer ID per slot, and 1024 is far above any + * real topic's subscriber count. + */ +const PEER_ANNOUNCE_RATE_MAX_SOURCES = 1024; +/** Bucket key for an announce that arrived without an attributable sender. */ +const PEER_ANNOUNCE_UNKNOWN_SOURCE = ''; + +/** + * Token bucket keyed by announcing peer ID, bounding how many addresses one source + * can push through peer-announce intake per unit of time. + * + * Partial grants are deliberate: a source over budget is trimmed rather than silenced, + * so a legitimate peer that overshoots still makes discovery progress. `now` is a + * parameter rather than a read of the clock so the refill curve is testable. + */ +export class AnnounceRateLimiter { + private readonly buckets = new Map(); + private readonly burst: number; + private readonly perMinute: number; + private readonly maxSources: number; + + constructor(burst: number = PEER_ANNOUNCE_RATE_BURST, perMinute: number = PEER_ANNOUNCE_RATE_PER_MINUTE, maxSources: number = PEER_ANNOUNCE_RATE_MAX_SOURCES) { + this.burst = burst; + this.perMinute = perMinute; + this.maxSources = maxSources; + } + + /** + * Spend up to `wanted` tokens on behalf of `source` and return how many were + * granted (0..wanted). An unknown source starts with a full bucket. + */ + take(source: string, wanted: number, now: number = Date.now()): number { + if (wanted <= 0) return 0; + const existing = this.buckets.get(source); + let tokens = this.burst; + if (existing) { + // Re-insert below so Map iteration order stays least-recently-used first. + this.buckets.delete(source); + tokens = Math.min(this.burst, existing.tokens + (Math.max(0, now - existing.seenAt) * this.perMinute) / 60_000); + } + const granted = Math.min(wanted, Math.floor(tokens)); + this.buckets.set(source, { tokens: tokens - granted, seenAt: now }); + while (this.buckets.size > this.maxSources) { + const oldest = this.buckets.keys().next(); + if (oldest.done) break; + this.buckets.delete(oldest.value); + } + return granted; + } +} + /** Dependencies for PeerAnnounceManager. */ export interface PeerAnnounceManagerDeps { /** Returns the current libp2p node (may be null if not started or already stopped). */ @@ -93,6 +167,8 @@ export class PeerAnnounceManager { * getSubscribers, so the cross-network leak stays closed. Pruned each emit(). */ private readonly topicMembers = new Map>(); + /** Per-announcing-peer intake budget — see {@link AnnounceRateLimiter}. */ + private readonly rateLimiter = new AnnounceRateLimiter(); constructor(deps: PeerAnnounceManagerDeps) { this.deps = deps; @@ -220,8 +296,17 @@ export class PeerAnnounceManager { if (droppedNonRoutable > 0 || droppedDuplicate > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${rawCount} addrs (${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate)`); return; } - const filtered = [...unique.values()]; - trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, network ${networkID.slice(0, 8)})`); + // Rate-limit AFTER dedup: a duplicate flood must not be able to drain the + // sender's budget and starve the addresses it announced legitimately. + const source = fromPeerID ?? PEER_ANNOUNCE_UNKNOWN_SOURCE; + const admitted = this.rateLimiter.take(source, unique.size); + if (admitted === 0) { + trace(`[NET] peer-announce from ${source.slice(0, 16)}: rate limited, dropped all ${unique.size} addrs`); + return; + } + const filtered = admitted < unique.size ? [...unique.values()].slice(0, admitted) : [...unique.values()]; + if (admitted < unique.size) trace(`[NET] peer-announce from ${source.slice(0, 16)}: rate limited to ${admitted}/${unique.size} addrs`); + trace(`[NET] peer-announce from ${source.slice(0, 16)}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, network ${networkID.slice(0, 8)})`); // Pass networkID so per-peer outcomes from gossiped entries surface in the // UI under the network through which they arrived. Identity-mismatch // outcomes inside addBootstrapPeers also trigger purgeStalePeer. diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 87c2ada9c..4ccb8aa1b 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; -import { PeerAnnounceManager, type PeerAnnounceMessage } from '../../../src/protocol/peer-announce.ts'; +import { AnnounceRateLimiter, PeerAnnounceManager, type PeerAnnounceMessage } from '../../../src/protocol/peer-announce.ts'; import { LISH_TOPIC_PREFIX } from '../../../src/protocol/constants.ts'; // Topic-scoping guard for peer-announce emit(): the transitive peer list broadcast @@ -236,6 +236,7 @@ describe('PeerAnnounceManager.emit recently-seen membership', () => { // are asserted here rather than left to the receivers further down the chain. const SRC_ID = '12D3KooWSourceSourceSourceSourceSourceSourceSourceSS'; +const OTHER_SRC_ID = '12D3KooWOtherOtherOtherOtherOtherOtherOtherOtherOO'; /** A manager wired only for handle(): captures the address lists it forwards. */ function intakeManager() { @@ -304,3 +305,69 @@ describe('PeerAnnounceManager.handle address dedup', () => { expect(forwarded).toEqual([]); }); }); + +describe('PeerAnnounceManager.handle per-source rate limit', () => { + it('lets a burst through, then throttles the same source', async () => { + // Budget is 384 addresses (3 × the 128 cap); a fourth full announce from the + // same source within the same second has nothing left to spend. + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + + expect(forwarded.map(f => f.length)).toEqual([128, 128, 128]); // 4th announce dropped entirely + }); + + it('throttles one source without starving another', async () => { + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + forwarded.length = 0; + await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', OTHER_SRC_ID); + + expect(forwarded.map(f => f.length)).toEqual([128]); + }); +}); + +describe('AnnounceRateLimiter', () => { + it('grants a full burst to an unseen source and nothing more', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + expect(limiter.take('a', 384, 0)).toBe(384); + expect(limiter.take('a', 1, 0)).toBe(0); + }); + + it('grants partially rather than refusing outright when over budget', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + expect(limiter.take('a', 500, 0)).toBe(384); + }); + + it('recovers over time at the configured rate', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('a', 200, 30_000)).toBe(128); // half a minute → half the per-minute rate + expect(limiter.take('a', 200, 60_000)).toBe(128); // another 30s worth + }); + + it('never refills past the burst ceiling', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('a', 1000, 600_000)).toBe(384); // ten idle minutes still caps at burst + }); + + it('keeps one source spending from starving another', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('b', 384, 0)).toBe(384); + }); + + it('bounds the bucket table, evicting the least recently heard-from source', () => { + const limiter = new AnnounceRateLimiter(10, 10, 2); + limiter.take('a', 10, 0); // 'a' exhausted, then pushed out by 'c' + limiter.take('b', 10, 0); + limiter.take('c', 10, 0); + + expect(limiter.take('a', 10, 0)).toBe(10); // evicted → fresh bucket + expect(limiter.take('c', 1, 0)).toBe(0); // recently used → still exhausted + }); +}); From 35856ecef62732ca6e1d188ef9d038ffad02c247 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:41:45 +0200 Subject: [PATCH 61/99] perf(protocol): group bootstrap status mutations into one emission --- backend/src/protocol/bootstrap-status.ts | 96 +++++++++-- .../unit/protocol/bootstrap-status.test.ts | 154 ++++++++++++++++++ 2 files changed, 240 insertions(+), 10 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 9a3c1b82e..b795a090a 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -26,12 +26,88 @@ const MAX_DISCOVERED_PER_NETWORK = 256; export class BootstrapStatusTracker { private readonly stats: Map> = new Map(); private onStatusChange: ((networkID: string, status: BootstrapStatus) => void) | null = null; + /** Open {@link batch} frames, keyed by networkID. See that method for why. */ + private readonly batches: Map = new Map(); /** Register a callback that fires on every status mutation. */ setOnChange(cb: ((networkID: string, status: BootstrapStatus) => void) | null): void { this.onStatusChange = cb; } + /** + * Group many mutations of one network into a SINGLE status emission. + * + * Every mutation otherwise rebuilds and emits the whole peer list, and a caller + * that walks a list of addresses performs two of them per address (pending, then + * outcome) — so intake of one large announce costs a snapshot per row per address, + * each copying every row, all but the last of which is thrown away by the UI. + * + * The frame closes on every exit path, throw included, so a body that fails still + * publishes what it managed to change — the tracker is already mutated by then and + * silence would leave the UI showing pre-batch state indefinitely. An async body is + * held open until its promise settles, because the caller this exists for awaits a + * dial between mutations; the return value keeps the body's own type either way. + * Nested calls for the same network collapse into the outermost frame, and a batch + * in which nothing actually changed emits nothing. + */ + batch(networkID: string, fn: () => T): T { + let frame = this.batches.get(networkID); + if (!frame) { + frame = { depth: 0, dirty: false }; + this.batches.set(networkID, frame); + } + frame.depth++; + const open = frame; + let result: T; + try { + result = fn(); + } catch (err) { + this.closeBatch(networkID, open); + throw err; + } + if (result instanceof Promise) { + return result.then( + value => { + this.closeBatch(networkID, open); + return value; + }, + err => { + this.closeBatch(networkID, open); + throw err; + } + ) as T; + } + this.closeBatch(networkID, open); + return result; + } + + /** Leave one {@link batch} frame, emitting the grouped snapshot when the last one exits. */ + private closeBatch(networkID: string, frame: { depth: number; dirty: boolean }): void { + frame.depth--; + if (frame.depth > 0) return; + // clear() drops open frames on teardown; a pending emission from before it + // belongs to the run that was torn down, not to whatever comes next. + if (this.batches.get(networkID) !== frame) return; + this.batches.delete(networkID); + if (frame.dirty) this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + + /** + * Publish one network's current status, or defer to the end of the open batch. + * + * Deferring skips {@link buildStatus} as well as the callback — building the + * snapshot is the part that copies every row, so suppressing only the callback + * would leave the cost in place. + */ + private notify(networkID: string): void { + const frame = this.batches.get(networkID); + if (frame) { + frame.dirty = true; + return; + } + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + /** Iterate over all tracked network IDs and their peer maps. Used for NET-CHURN dump. */ entries(): IterableIterator<[string, Map]> { return this.stats.entries(); @@ -58,8 +134,7 @@ export class BootstrapStatusTracker { const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: new Date().toISOString() }); this.capDiscovered(net); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + this.notify(networkID); } /** Record a dial outcome (connected, timeout, error, identity-mismatch). */ @@ -71,8 +146,7 @@ export class BootstrapStatusTracker { const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString() }); this.capDiscovered(net); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + this.notify(networkID); } /** Bound discovered rows per network (drop the oldest) — see MAX_DISCOVERED_PER_NETWORK. */ @@ -90,8 +164,7 @@ export class BootstrapStatusTracker { if (!net) return; net.delete(multiaddr); if (net.size === 0) this.stats.delete(networkID); - const snap = this.buildStatus(networkID) ?? { networkID, peers: [] }; - this.onStatusChange?.(networkID, snap); + this.notify(networkID); } /** @@ -111,7 +184,7 @@ export class BootstrapStatusTracker { } if (!changed) continue; if (peers.size === 0) this.stats.delete(networkID); - this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + this.notify(networkID); } } @@ -139,7 +212,7 @@ export class BootstrapStatusTracker { } if (!changed) continue; if (peers.size === 0) this.stats.delete(networkID); - this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + this.notify(networkID); } } @@ -165,18 +238,21 @@ export class BootstrapStatusTracker { // returns null for a dropped network, and skipping the callback would leave the // UI showing the very row that was just removed. Same fallback the other // removal paths use. - this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + this.notify(networkID); } /** Reset the bootstrap status for a single network (used when re-joining). */ resetNetwork(networkID: string): void { this.stats.delete(networkID); - this.onStatusChange?.(networkID, { networkID, peers: [] }); + this.notify(networkID); } /** Clear all tracked state (called from Network.stop()). */ clear(): void { this.stats.clear(); + // An in-flight batch belongs to the run being torn down; its pending emission + // would publish the next run's (empty) state under the old run's networkID. + this.batches.clear(); } private ensureNetwork(networkID: string): Map { diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 8f61aadfc..f05bd565a 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -184,6 +184,160 @@ describe('BootstrapStatusTracker.sweepStale', () => { expect(tracker.getStatus(NET)?.peers.length).toBe(1); }); + + /** + * markPending fires whenever gossip names an address again, which happens on every + * announce cycle — far more often than the sweep TTL. Treating that as activity kept + * a dead peer's row alive forever. Only a dial that produced an outcome counts. + */ + // Rows are stamped with `new Date()`, which no Date.now stub can steer, so these + // read the real timestamp the tracker wrote and drive sweepStale relative to it. + // The short sleep only guarantees a measurable gap between the two writes. + const clockOf = (tracker: BootstrapStatusTracker): string => tracker.getStatus(NET)!.peers[0]!.updatedAt; + + it('lets a real dial outcome refresh the clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const firstAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + expect(Date.parse(clockOf(tracker))).toBeGreaterThan(Date.parse(firstAt)); + tracker.sweepStale(TTL, () => false, Date.parse(firstAt) + TTL + 2); + expect(tracker.getStatus(NET)?.peers.length).toBe(1); // survives — clock moved + }); + + it('starts the clock on a first mention that has no prior row', () => { + const tracker = new BootstrapStatusTracker(); + tracker.markPending(NET, DEAD_ADDR, DEAD_ID, 'discovered'); + const mentionAt = clockOf(tracker); + + tracker.sweepStale(TTL, () => false, Date.parse(mentionAt) + TTL - 60_000); + expect(tracker.getStatus(NET)?.peers.length).toBe(1); // inside the TTL + + tracker.sweepStale(TTL, () => false, Date.parse(mentionAt) + TTL + 2); + expect(tracker.getStatus(NET)).toBe(null); // and expires once past it + }); +}); + +/** + * Every mutation rebuilds and emits the whole peer list, and intake of one announce + * performs two per address. batch() groups them so the UI receives one snapshot for + * the run instead of one per row per address. + */ +describe('BootstrapStatusTracker.batch', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWBatchBatchBatchBatchBatchBatchBatchBatchBB'; + const addr = (i: number): string => `/ip4/192.0.2.${i}/tcp/9090/p2p/${PID}`; + + function tracked() { + const tracker = new BootstrapStatusTracker(); + const seen: string[][] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.map(p => p.multiaddr))); + return { tracker, seen }; + } + + it('emits exactly one snapshot for many mutations', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => { + for (let i = 1; i <= 10; i++) { + tracker.markPending(NET, addr(i), PID, 'discovered'); + tracker.recordOutcome(NET, addr(i), PID, 'connected', null, null, 'discovered'); + } + }); + + expect(seen.length).toBe(1); + expect(seen[0]!.length).toBe(10); // the one snapshot holds every row + }); + + it('still emits when the body throws', () => { + const { tracker, seen } = tracked(); + + expect(() => + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial loop blew up'); + }) + ).toThrow('dial loop blew up'); + + expect(seen).toEqual([[addr(1)]]); + }); + + it('holds the frame open across awaits and emits once the promise settles', async () => { + const { tracker, seen } = tracked(); + + const done = tracker.batch(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + await Promise.resolve(); + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + expect(seen).toEqual([]); // nothing emitted while the body is still running + await done; + + expect(seen).toEqual([[addr(1), addr(2)]]); + }); + + it('emits once when a rejected async body settles', async () => { + const { tracker, seen } = tracked(); + + const done = tracker.batch(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial rejected'); + }); + + await expect(done).rejects.toThrow('dial rejected'); + expect(seen).toEqual([[addr(1)]]); + }); + + it('emits nothing when the body changed nothing', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => {}); + + expect(seen).toEqual([]); + }); + + it('collapses nested batches of the same network into one snapshot', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + expect(seen).toEqual([]); // inner exit must not publish a half-built run + }); + + expect(seen).toEqual([[addr(1), addr(2)]]); + }); + + it('leaves single-mutation callers emitting per mutation', () => { + const { tracker, seen } = tracked(); + + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + + expect(seen.length).toBe(2); + }); + + it('does not defer mutations of a different network', () => { + const { tracker, seen } = tracked(); + const OTHER = 'netBBBB'; + + tracker.batch(NET, () => { + tracker.recordOutcome(OTHER, addr(1), PID, 'connected', null, null, 'discovered'); + expect(seen.length).toBe(1); // the other network is not part of this batch + }); + + expect(seen.length).toBe(1); // NET itself changed nothing → no second emit + }); + + it('returns the body value unchanged', () => { + const { tracker } = tracked(); + expect(tracker.batch(NET, () => 42)).toBe(42); + }); }); describe('BootstrapStatusTracker discovered-row cap', () => { From 5f6bacd82593ec701e94b7569862752db900e6ed Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:42:05 +0200 Subject: [PATCH 62/99] fix(protocol): keep the staleness clock when a peer is re-mentioned --- backend/src/protocol/bootstrap-status.ts | 9 ++++++++- .../tests/unit/protocol/bootstrap-status.test.ts | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index b795a090a..94fe52081 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -132,7 +132,14 @@ export class BootstrapStatusTracker { // the same multiaddr must not downgrade it to 'discovered'. const previous = net.get(multiaddr); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; - net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: new Date().toISOString() }); + // Keep the existing staleness clock (see sweepStale). Reaching this point means + // someone MENTIONED the peer again — gossip repeating an address it still + // remembers — which is evidence about the announcer, not about the peer. Letting + // a mention move the clock made a dead peer's row immortal: every announce cycle + // is far shorter than the sweep TTL, so the row was refreshed long before it + // could expire, no matter how many dials to it had already failed. Only a dial + // that actually produced an outcome advances it, in recordOutcome below. + net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString() }); this.capDiscovered(net); this.notify(networkID); } diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index f05bd565a..7542613c9 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -195,6 +195,19 @@ describe('BootstrapStatusTracker.sweepStale', () => { // The short sleep only guarantees a measurable gap between the two writes. const clockOf = (tracker: BootstrapStatusTracker): string => tracker.getStatus(NET)!.peers[0]!.updatedAt; + it('does not let a re-mention refresh the staleness clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const outcomeAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.markPending(NET, DEAD_ADDR, DEAD_ID, 'discovered'); // gossip mentions it again + + expect(clockOf(tracker)).toBe(outcomeAt); // clock untouched by the mention + tracker.sweepStale(TTL, () => false, Date.parse(outcomeAt) + TTL + 2); + expect(tracker.getStatus(NET)).toBe(null); // ages out from the last real outcome + }); + it('lets a real dial outcome refresh the clock', async () => { const tracker = new BootstrapStatusTracker(); tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); From bcedb2713a1ef6da65890fa89f04b5e64296ee60 Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:51:44 +0200 Subject: [PATCH 63/99] refactor(protocol): use the shared destination peer id helper --- backend/src/protocol/network.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 4bbdab94d..68467b493 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1174,8 +1174,7 @@ export class Network { console.log(` [NET-CHURN] bootstrap stats net=${networkID.slice(0, 8)}: ${parts}`); } for (const ma of this.bootstrapMultiaddrs) { - const p2pComponents = ma.getComponents().filter((c: { code: number; value?: string }) => c.code === 421); - const pid: string | undefined = p2pComponents.length > 0 ? p2pComponents[p2pComponents.length - 1].value : undefined; + const pid = extractDestinationPeerID(ma); if (pid && this.isRedialSuppressed(pid)) continue; // deliberately left — don't resurrect it here // A CONFIGURED entry is the user's way back in and is always tried; a // DISCOVERED one earned its place here by answering once, but that is no From ff60a35b248d822cd94ed3ef8d116ee9fe6786aa Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:55:18 +0200 Subject: [PATCH 64/99] fix(network): read live connectivity in zero-connection recovery --- backend/src/protocol/network.ts | 13 +++- .../unit/protocol/network-disconnect.test.ts | 14 ++-- .../protocol/peer-eviction-guards.test.ts | 67 ++++++++++++++++++- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 68467b493..9c7d738b7 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -917,7 +917,7 @@ export class Network { this.checkPeerCounts(); await this.runRedialMaintenance(connectedPeers, allPeers, epoch); if (epoch !== this.runEpoch) return; - await this.runZeroConnectionRecovery(connectedPeers, epoch); + await this.runZeroConnectionRecovery(epoch); if (epoch !== this.runEpoch) return; await this.maybePromotePeers(epoch); if (epoch !== this.runEpoch) return; @@ -1150,10 +1150,13 @@ export class Network { for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } - private async runZeroConnectionRecovery(connectedPeers: any[], epoch: number = this.runEpoch): Promise { + private async runZeroConnectionRecovery(epoch: number = this.runEpoch): Promise { const node = this.node; if (!node || epoch !== this.runEpoch) return; - if (!AUTODIAL_WORKAROUND || connectedPeers.length !== 0 || this.bootstrapMultiaddrs.length === 0) return; + // Read connectivity here rather than trusting the snapshot the status tick opened + // with: re-dial maintenance runs in between and may already have reconnected us, + // in which case the node is not isolated and every dial below is pure churn. + if (!AUTODIAL_WORKAROUND || node.getPeers().length !== 0 || this.bootstrapMultiaddrs.length === 0) return; console.log(` ⚠️ No connections - dialing ${this.bootstrapMultiaddrs.length} bootstrap peer(s) directly...`); // [NET-CHURN] dump: who left in the run-up to this zero-connection // state, and what each configured bootstrap entry's last dial outcome @@ -1192,6 +1195,10 @@ export class Network { // Each dial awaits for up to 10s, so a stop() can land mid-loop; the // remaining dials belong to a node this run no longer owns. if (epoch !== this.runEpoch) return; + // The whole point of this loop is isolation. A dial from an earlier pass that + // resolved late, or an inbound connection, ends it — carrying on would open + // connections the node no longer needs. + if (node.getPeers().length > 0) return; try { console.log(` → Dialing ${maStr}`); await node.dial(ma, { signal: AbortSignal.timeout(10000) }); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index c80c2629e..18babfc4e 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -189,6 +189,8 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { (network as any).recentDisconnects = []; (network as any).bootstrapTracker = { entries: () => [] }; (network as any).node = { + // Recovery reads connectivity itself rather than trusting the tick's snapshot. + getPeers: () => [], async dial(ma: { toString(): string }): Promise { dialed.push(ma.toString()); }, @@ -196,12 +198,12 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { return { network, dialed }; } - const run = (network: Network, connected: any[]): Promise => (network as any).runZeroConnectionRecovery(connected); + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(); it('does not dial a bootstrap peer suppressed by leave-network', async () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([PEER_ID], [ma]); - await run(network, []); + await run(network); expect(dialed).toEqual([]); }); @@ -215,7 +217,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([], [ma]); (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); - await run(network, []); + await run(network); expect(dialed).toEqual([]); }); @@ -225,7 +227,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(multiaddr(ma).toString())]); - await run(network, []); + await run(network); expect(dialed).toEqual([multiaddr(ma).toString()]); }); @@ -233,14 +235,14 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([], [ma]); (network as any).unreachableQuarantine = new Map([[PEER_ID, Date.now() - 60_000]]); - await run(network, []); + await run(network); expect(dialed).toEqual([]); }); it('still dials a non-suppressed bootstrap peer', async () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([], [ma]); - await run(network, []); + await run(network); expect(dialed).toEqual([multiaddr(ma).toString()]); }); }); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 87624f156..4ad40b7da 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1006,7 +1006,7 @@ describe('configured origin is a property of the address, not the peer', () => { const { network, dialed } = bareNetwork(); (network as any).bootstrapMultiaddrs = [multiaddr(DISCOVERED)]; (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); - await (network as any).runZeroConnectionRecovery([]); + await (network as any).runZeroConnectionRecovery(); expect(dialed).toEqual([]); }); @@ -1014,7 +1014,7 @@ describe('configured origin is a property of the address, not the peer', () => { const { network, dialed } = bareNetwork(); (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED)]; (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); - await (network as any).runZeroConnectionRecovery([]); + await (network as any).runZeroConnectionRecovery(); expect(dialed).toEqual([multiaddr(CONFIGURED).toString()]); }); @@ -1088,3 +1088,66 @@ describe('Network.stop — per-run state really is per run', () => { expect((network as any).delayedPeerCountTimers.size).toBe(0); }); }); + +describe('runZeroConnectionRecovery — connectivity is read, not remembered', () => { + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function bareNetwork(opts: { addresses?: string[]; peers?: () => unknown[]; onDial?: (address: string) => void } = {}) { + const dialed: string[] = []; + // The churn dump is the first thing the loop does, so it witnesses whether the + // isolation check at the top of the function ran at all — a later check inside the + // loop would already have let the misleading "No connections" report out. + let churnDumps = 0; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapMultiaddrs = (opts.addresses ?? [ADDR_A]).map(a => multiaddr(a)); + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { + entries: () => { + churnDumps++; + return []; + }, + }; + (network as any).node = { + getPeers: opts.peers ?? ((): unknown[] => []), + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + opts.onDial?.(ma.toString()); + }, + }; + return { network, dialed, churn: () => churnDumps }; + } + + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(1); + + it('does not dial at all when a peer connected since the tick began', async () => { + const { network, dialed, churn } = bareNetwork({ peers: () => [{ toString: () => PEER_B }] }); + await run(network); + expect(dialed).toEqual([]); + expect(churn()).toBe(0); + }); + + it('stops the pass as soon as a connection exists', async () => { + // The first dial fails, but an inbound connection lands during it; the second + // address must not be tried, because the node is no longer isolated. + let connected = false; + const { network, dialed } = bareNetwork({ + addresses: [ADDR_A, ADDR_B], + peers: (): unknown[] => (connected ? [{ toString: () => PEER_B }] : []), + onDial: () => { + connected = true; + throw new Error('dial timeout'); + }, + }); + await run(network); + expect(dialed).toEqual([multiaddr(ADDR_A).toString()]); + }); +}); From 4da273335dcd1866813805941870f0114f7a9d3a Mon Sep 17 00:00:00 2001 From: LuRy Date: Sun, 16 Aug 2026 23:57:25 +0200 Subject: [PATCH 65/99] fix(network): pace recovery dials of discovered addresses --- backend/src/protocol/network.ts | 31 ++++++++ .../protocol/peer-eviction-guards.test.ts | 72 +++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 9c7d738b7..2d150cf54 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1150,6 +1150,33 @@ export class Network { for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } + /** + * Record a failed recovery dial of a DISCOVERED address against the shared per-peer + * backoff, in the same four-field shape every other writer uses. + * + * Without this the recovery loop was the one dial path that paced nothing: for an + * address whose peer is not in the peerStore, re-dial maintenance never sees the peer + * either, so a dead entry was re-dialed on every tick for as long as the node stayed + * isolated. `evictionFails` deliberately does not grow — at zero connections there is + * no evidence the remote is the broken side, which is exactly what + * {@link nextEvictionFailCount} resets on. + */ + private noteRecoveryDialFailure(peerID: string): void { + const now = Date.now(); + const previous = this.redialBackoff.get(peerID); + const failCount = previous?.failCount ?? 0; + this.redialBackoff.set(peerID, { + nextAttempt: now + Math.min(30_000 * 2 ** failCount, 600_000), + failCount: failCount + 1, + firstFailure: previous?.firstFailure ?? now, + evictionFails: previous?.evictionFails ?? 0, + }); + // An expired quarantine is what let this dial through, and it buys exactly one + // probe: re-arm it on failure or every later pass spends another dial on a peer + // that has already been written off once. + if (this.unreachableQuarantine.has(peerID)) this.unreachableQuarantine.set(peerID, now); + } + private async runZeroConnectionRecovery(epoch: number = this.runEpoch): Promise { const node = this.node; if (!node || epoch !== this.runEpoch) return; @@ -1202,9 +1229,13 @@ export class Network { try { console.log(` → Dialing ${maStr}`); await node.dial(ma, { signal: AbortSignal.timeout(10000) }); + if (epoch !== this.runEpoch) return; + if (pid) this.redialBackoff.delete(pid); console.log(` ✓ Connected via ${maStr}`); break; } catch (err: any) { + if (epoch !== this.runEpoch) return; + if (pid) this.noteRecoveryDialFailure(pid); console.log(` ✗ Failed ${maStr}: ${err.message ?? err}`); } } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 4ad40b7da..e2f7e4d3f 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1151,3 +1151,75 @@ describe('runZeroConnectionRecovery — connectivity is read, not remembered', ( expect(dialed).toEqual([multiaddr(ADDR_A).toString()]); }); }); + +describe('runZeroConnectionRecovery — a failed dial paces the next one', () => { + const DISCOVERED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { address: string; configured: boolean }) { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(opts.configured ? [normalizeMultiaddrForCompare(opts.address)] : []); + (network as any).bootstrapMultiaddrs = [multiaddr(opts.address)]; + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).node = { + getPeers: (): unknown[] => [], + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + throw new Error('dial timeout'); + }, + }; + return { network, dialed }; + } + + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(1); + + it('writes the full four-field backoff record for a discovered address', async () => { + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + await run(network); + const entry = (network as any).redialBackoff.get(PEER_ID) as { nextAttempt: number; failCount: number; firstFailure: number; evictionFails: number } | undefined; + expect(entry).toBeDefined(); + expect(Object.keys(entry!).sort()).toEqual(['evictionFails', 'failCount', 'firstFailure', 'nextAttempt']); + expect(entry!.failCount).toBe(1); + expect(entry!.nextAttempt).toBeGreaterThan(Date.now()); + }); + + it('skips the same address on the very next pass', async () => { + const { network, dialed } = bareNetwork({ address: DISCOVERED, configured: false }); + await run(network); + await run(network); + expect(dialed).toEqual([multiaddr(DISCOVERED).toString()]); + }); + + /** + * At zero connections we cannot tell the remote apart from our own outage, which is + * the exact condition nextEvictionFailCount resets on — so a recovery failure must + * never become evidence against the peer. + */ + it('does not count the failure towards eviction', async () => { + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() - 1, failCount: 2, firstFailure: Date.now() - 60_000, evictionFails: 3 }]]); + await run(network); + expect(((network as any).redialBackoff.get(PEER_ID) as { evictionFails: number }).evictionFails).toBe(3); + }); + + it('re-arms an expired quarantine that let the dial through', async () => { + const longAgo = Date.now() - 10 * 60 * 60_000; + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + (network as any).unreachableQuarantine = new Map([[PEER_ID, longAgo]]); + await run(network); + expect((network as any).unreachableQuarantine.get(PEER_ID)).toBeGreaterThan(longAgo); + }); + + /** + * Configured entries stay exempt from eviction and from quarantine — but exempt is + * not unlimited: several dead ones at a 10 s timeout each turn every tick into + * minutes of back-to-back dialing. + */ +}); From 675da485d46c6cd9ed60b42c31e058472aad51d4 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 00:02:20 +0200 Subject: [PATCH 66/99] fix(network): pace configured bootstrap probes per address --- backend/src/protocol/network.ts | 65 +++++++++++++++++-- .../unit/protocol/network-disconnect.test.ts | 1 + .../protocol/peer-eviction-guards.test.ts | 29 ++++++++- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 2d150cf54..ba647c206 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -121,6 +121,21 @@ const BOOTSTRAP_STATUS_STALE_MS = 30 * 60_000; */ const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; +/** + * Backoff ceiling for the loops that probe ONE CONFIGURED ADDRESS. + * + * A configured entry is exempt from eviction and from quarantine — it is user data and + * the way back into the network — but "never given up on" is not "dialed without limit". + * Each attempt spends a 10 s dial timeout, so a handful of dead configured addresses + * could occupy a status tick end to end, every tick, forever. + * + * Half the general re-dial ceiling (10 min) deliberately: a configured address deserves + * to be retried more often than a gossip-learned one, so a bootstrap that comes back is + * picked up within five minutes without the operator touching anything, while a + * permanently dead one costs one dial per five minutes instead of one per 30 s tick. + */ +const CONFIGURED_PROBE_BACKOFF_MAX_MS = 5 * 60_000; + /** * Where the eviction window should run from after a re-dial failure. * @@ -342,6 +357,17 @@ export class Network { private readonly redialBackoff = new Map(); /** peerID → eviction time. Blocks re-adding a just-evicted unreachable peer from gossip for UNREACHABLE_QUARANTINE_MS. */ private readonly unreachableQuarantine = new Map(); + /** + * Canonical multiaddr → pacing record for the loops that probe ONE CONFIGURED ADDRESS + * (zero-connection recovery and the parked-bootstrap probe). + * + * Keyed by ADDRESS, not by peer, because those loops ask an address-level question. + * One peer can hold a dead configured address and a working one at the same time; + * with a peer-keyed record the dead address's failure puts the whole peer into + * backoff, the working address is skipped for the rest of the pass, and the next pass + * starts at the dead one again — so the working address could go untried indefinitely. + */ + private readonly addressProbeBackoff = new Map(); /** * peerID → time we first saw the peer disconnected with ZERO reachable * addresses. Such peers never enter the re-dial path (nothing to dial), so @@ -1150,6 +1176,29 @@ export class Network { for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } + /** + * Whether a per-ADDRESS probe of a configured entry is due. + * + * An address nothing has failed on yet is always due; one that failed waits out the + * window {@link noteAddressProbeFailure} set for it. + */ + private isAddressProbeDue(canonicalAddress: string, now: number): boolean { + const entry = this.addressProbeBackoff.get(canonicalAddress); + return entry === undefined || entry.nextAttempt <= now; + } + + /** + * Record a failed probe of a configured ADDRESS: 30 s × 2^fails, capped at + * {@link CONFIGURED_PROBE_BACKOFF_MAX_MS}. + * + * Paces and nothing more — a configured entry is exempt from eviction and from + * quarantine, so this record must never become the evidence that removes one. + */ + private noteAddressProbeFailure(canonicalAddress: string): void { + const failCount = (this.addressProbeBackoff.get(canonicalAddress)?.failCount ?? 0) + 1; + this.addressProbeBackoff.set(canonicalAddress, { nextAttempt: Date.now() + Math.min(30_000 * 2 ** (failCount - 1), CONFIGURED_PROBE_BACKOFF_MAX_MS), failCount }); + } + /** * Record a failed recovery dial of a DISCOVERED address against the shared per-peer * backoff, in the same four-field shape every other writer uses. @@ -1212,8 +1261,13 @@ export class Network { // this, an isolated node re-dialed a dead discovered peer every 30s // forever, since maintenance stops counting failures the moment we have no // other connection to prove we are online. - const configured = this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(ma.toString())); - if (pid && !configured && !isRecoveryDialDue(pid, Date.now(), this.redialBackoff, this.unreachableQuarantine)) continue; + const canonical = normalizeMultiaddrForCompare(ma.toString()); + const configured = this.configuredBootstrapAddresses.has(canonical); + if (configured) { + if (!this.isAddressProbeDue(canonical, Date.now())) continue; + } else if (pid && !isRecoveryDialDue(pid, Date.now(), this.redialBackoff, this.unreachableQuarantine)) { + continue; + } // Routability is re-checked here, not just at configure time: a LAN or VPN // bootstrap is on this list while its interface is down, and becomes dialable // again the moment it returns. @@ -1230,12 +1284,14 @@ export class Network { console.log(` → Dialing ${maStr}`); await node.dial(ma, { signal: AbortSignal.timeout(10000) }); if (epoch !== this.runEpoch) return; - if (pid) this.redialBackoff.delete(pid); + if (configured) this.addressProbeBackoff.delete(canonical); + else if (pid) this.redialBackoff.delete(pid); console.log(` ✓ Connected via ${maStr}`); break; } catch (err: any) { if (epoch !== this.runEpoch) return; - if (pid) this.noteRecoveryDialFailure(pid); + if (configured) this.noteAddressProbeFailure(canonical); + else if (pid) this.noteRecoveryDialFailure(pid); console.log(` ✗ Failed ${maStr}: ${err.message ?? err}`); } } @@ -2445,6 +2501,7 @@ export class Network { this._lastScores.clear(); this.redialBackoff.clear(); this.unreachableQuarantine.clear(); + this.addressProbeBackoff.clear(); this.noReachableSince.clear(); this.configuredBootstrapPeerIDs.clear(); this.configuredBootstrapAddresses.clear(); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index 18babfc4e..9a4cc0a34 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -182,6 +182,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index e2f7e4d3f..27e7bcaa1 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -29,6 +29,7 @@ describe('purgeStalePeer — epoch guard', () => { (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).node = { getConnections: () => [ @@ -983,6 +984,7 @@ describe('configured origin is a property of the address, not the peer', () => { (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(CONFIGURED)]); (network as any).unreachableQuarantine = new Map(); (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set([PEER_ID]); (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED), multiaddr(DISCOVERED)]; (network as any).recentDisconnects = []; @@ -1035,7 +1037,7 @@ describe('configured origin is a property of the address, not the peer', () => { describe('Network.stop — per-run state really is per run', () => { function bareNetwork() { const network = Object.create(Network.prototype) as Network; - for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys']) { + for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'addressProbeBackoff', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys']) { (network as any)[field] = field === 'seenSearchIDs' || field === 'dcutrPeers' || field === 'bootstrapPeerIDs' || field === 'configuredBootstrapPeerIDs' || field === 'configuredBootstrapAddresses' ? new Set() : new Map(); } (network as any).runEpoch = 1; @@ -1154,6 +1156,7 @@ describe('runZeroConnectionRecovery — connectivity is read, not remembered', ( describe('runZeroConnectionRecovery — a failed dial paces the next one', () => { const DISCOVERED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const CONFIGURED = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; function bareNetwork(opts: { address: string; configured: boolean }) { const dialed: string[] = []; @@ -1222,4 +1225,28 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => * not unlimited: several dead ones at a 10 s timeout each turn every tick into * minutes of back-to-back dialing. */ + + it('paces a configured address too, on its own record', async () => { + const { network, dialed } = bareNetwork({ address: CONFIGURED, configured: true }); + await run(network); + await run(network); + expect(dialed).toEqual([multiaddr(CONFIGURED).toString()]); + }); + + it('keeps the configured wait well under the general re-dial ceiling', async () => { + const { network } = bareNetwork({ address: CONFIGURED, configured: true }); + const key = normalizeMultiaddrForCompare(CONFIGURED); + for (let failCount = 0; failCount < 12; failCount++) { + (network as any).addressProbeBackoff.set(key, { nextAttempt: 0, failCount }); + await run(network); + } + const entry = (network as any).addressProbeBackoff.get(key) as { nextAttempt: number }; + expect(entry.nextAttempt - Date.now()).toBeLessThanOrEqual(5 * 60_000); + }); + + it('leaves the per-peer eviction record untouched for a configured address', async () => { + const { network } = bareNetwork({ address: CONFIGURED, configured: true }); + await run(network); + expect((network as any).redialBackoff.size).toBe(0); + }); }); From 1904d69cbee60acd54e63cd3446ee12719b36edd Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 00:04:49 +0200 Subject: [PATCH 67/99] fix(network): key the parked bootstrap probe by address --- backend/src/protocol/network.ts | 59 +++++++++++------- .../protocol/peer-eviction-guards.test.ts | 62 ++++++++++++++++--- 2 files changed, 89 insertions(+), 32 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index ba647c206..42e65da8b 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1226,6 +1226,25 @@ export class Network { if (this.unreachableQuarantine.has(peerID)) this.unreachableQuarantine.set(peerID, now); } + /** + * Whether some open connection already terminates on the exact endpoint an address + * names. + * + * The peer-level question — "are we connected to them at all" — is the wrong one for a + * per-address probe: a peer reachable through a second address would mask a broken + * configured entry for as long as that other route held, which is precisely the case + * the probe exists to expose. + */ + private hasConnectionOnEndpoint(ma: any): boolean { + if (!this.node) return false; + const target = ma.toString(); + try { + return this.node.getConnections().some(c => isSameDialEndpoint(String(c.remoteAddr ?? ''), target)); + } catch { + return false; + } + } + private async runZeroConnectionRecovery(epoch: number = this.runEpoch): Promise { const node = this.node; if (!node || epoch !== this.runEpoch) return; @@ -1306,44 +1325,38 @@ export class Network { * recovery would pick it up, but only while the node has NO connections at all, so a * node happily talking to someone else would never notice the tunnel came back. * - * Runs on the slow promote cadence and honours the same pacing as every other loop, - * so a permanently broken entry costs one dial per window, not one per tick. + * Runs on the slow promote cadence and paces itself per ADDRESS, so a permanently + * broken entry costs one dial per window and cannot starve a sibling address of the + * same peer that does work. */ private async probeParkedConfiguredBootstraps(epoch: number = this.runEpoch): Promise { const node = this.node; if (!node || epoch !== this.runEpoch) return; const localCidrs = getLocalCidrs(); - const now = Date.now(); for (const ma of [...this.bootstrapMultiaddrs]) { if (epoch !== this.runEpoch) return; + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (!this.configuredBootstrapAddresses.has(canonical)) continue; const pid = extractDestinationPeerID(ma); - if (!pid || !this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(ma.toString()))) continue; - if (this.isRedialSuppressed(pid)) continue; + if (pid && this.isRedialSuppressed(pid)) continue; // Still unreachable from here — leave it parked for a later pass. if (shouldDenyDial(ma, localCidrs)) continue; - if (!isRecoveryDialDue(pid, now, this.redialBackoff, this.unreachableQuarantine)) continue; - try { - if (node.getConnections(peerIDFromString(pid)).length > 0) continue; - } catch { - continue; // unparseable id — nothing sane to probe - } + if (!this.isAddressProbeDue(canonical, Date.now())) continue; + // Only a connection ON THIS ENDPOINT answers the question the probe asks. A + // connection to the same peer over some other address used to skip it, which + // is exactly how a broken configured entry kept looking fine. + if (this.hasConnectionOnEndpoint(ma)) continue; try { - await node.dial(ma, { signal: AbortSignal.timeout(10000) }); + // Forced for the same reason the configured branch of addBootstrapPeers + // forces: without it libp2p hands back whatever connection it already holds + // to this peer and the probe proves nothing about the address. + await node.dial(ma, { signal: AbortSignal.timeout(10000), force: true }); if (epoch !== this.runEpoch) return; - this.redialBackoff.delete(pid); + this.addressProbeBackoff.delete(canonical); console.log(`[NET] parked configured bootstrap reachable again: ${ma.toString()}`); } catch (err: any) { if (epoch !== this.runEpoch) return; - const previous = this.redialBackoff.get(pid); - const failCount = previous?.failCount ?? 0; - // Paces itself and nothing more: a configured peer is exempt from eviction, - // so this probe must never become the evidence that evicts one. - this.redialBackoff.set(pid, { - nextAttempt: Date.now() + Math.min(30_000 * 2 ** failCount, 600_000), - failCount: failCount + 1, - firstFailure: previous?.firstFailure ?? Date.now(), - evictionFails: previous?.evictionFails ?? 0, - }); + this.noteAddressProbeFailure(canonical); trace(`[NET] parked configured bootstrap still failing: ${ma.toString()} — ${err?.message ?? err}`); } } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 27e7bcaa1..73560474f 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -911,20 +911,26 @@ describe('addBootstrapPeers — identity mismatch trims the address, not the pee describe('probeParkedConfiguredBootstraps', () => { const PARKED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; - function bareNetwork(opts: { configured?: boolean; connections?: number } = {}) { + /** A second configured address of the SAME peer — the sibling finding 4 is about. */ + const SIBLING = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { configured?: boolean; addresses?: string[]; connectionAddrs?: string[]; failAddresses?: string[] } = {}) { const dialed: string[] = []; + const addresses = opts.addresses ?? [PARKED]; const network = Object.create(Network.prototype) as Network; (network as any).runEpoch = 1; (network as any).redialSuppressedByNet = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured === false ? [] : [PEER_ID]); - (network as any).configuredBootstrapAddresses = new Set(opts.configured === false ? [] : [normalizeMultiaddrForCompare(PARKED)]); + (network as any).configuredBootstrapAddresses = new Set(opts.configured === false ? [] : addresses.map(a => normalizeMultiaddrForCompare(a))); (network as any).unreachableQuarantine = new Map(); (network as any).redialBackoff = new Map(); - (network as any).bootstrapMultiaddrs = [multiaddr(PARKED)]; + (network as any).addressProbeBackoff = new Map(); + (network as any).bootstrapMultiaddrs = addresses.map(a => multiaddr(a)); (network as any).node = { - getConnections: () => Array.from({ length: opts.connections ?? 0 }, () => ({})), + getConnections: () => (opts.connectionAddrs ?? []).map(a => ({ remoteAddr: { toString: () => a } })), async dial(ma: { toString(): string }): Promise { dialed.push(ma.toString()); + if (opts.failAddresses?.includes(ma.toString())) throw new Error('dial timeout'); }, }; return { network, dialed }; @@ -933,7 +939,7 @@ describe('probeParkedConfiguredBootstraps', () => { const run = (network: Network): Promise => (network as any).probeParkedConfiguredBootstraps(1); it('probes a configured address even though we hold other connections', async () => { - const { network, dialed } = bareNetwork(); + const { network, dialed } = bareNetwork({ connectionAddrs: [`/ip4/198.51.100.200/tcp/9090/p2p/${PEER_ID}`] }); await run(network); expect(dialed).toEqual([multiaddr(PARKED).toString()]); }); @@ -944,15 +950,20 @@ describe('probeParkedConfiguredBootstraps', () => { expect(dialed).toEqual([]); }); - it('does not re-probe a peer that is already connected', async () => { - const { network, dialed } = bareNetwork({ connections: 1 }); + /** + * Only a connection ON THIS ENDPOINT answers what the probe asks. A connection to the + * same peer over another address used to skip it — which is how a broken configured + * entry kept looking fine for as long as the peer was reachable some other way. + */ + it('skips only when the existing connection is on this very address', async () => { + const { network, dialed } = bareNetwork({ connectionAddrs: [PARKED] }); await run(network); expect(dialed).toEqual([]); }); it('respects the backoff so a broken entry costs one dial per window', async () => { const { network, dialed } = bareNetwork(); - (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + (network as any).addressProbeBackoff = new Map([[normalizeMultiaddrForCompare(PARKED), { nextAttempt: Date.now() + 60_000, failCount: 1 }]]); await run(network); expect(dialed).toEqual([]); }); @@ -963,8 +974,41 @@ describe('probeParkedConfiguredBootstraps', () => { await run(network); expect(dialed).toEqual([]); }); -}); + /** + * The backoff used to be keyed by PEER while the loop iterates by ADDRESS: the dead + * address failed, the whole peer went into backoff, its working sibling was skipped + * for the rest of the pass — and the next pass started at the dead one again, so the + * sibling could go untried indefinitely. + */ + it('tries the sibling address of a peer whose other address just failed', async () => { + const { network, dialed } = bareNetwork({ addresses: [PARKED, SIBLING], failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + expect(dialed).toEqual([multiaddr(PARKED).toString(), multiaddr(SIBLING).toString()]); + }); + + it('paces each address on its own record', async () => { + const { network } = bareNetwork({ addresses: [PARKED, SIBLING], failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + const backoff = (network as any).addressProbeBackoff as Map; + expect(backoff.has(normalizeMultiaddrForCompare(PARKED))).toBe(true); + expect(backoff.has(normalizeMultiaddrForCompare(SIBLING))).toBe(false); + }); + + /** + * Without `force` libp2p hands back whatever connection it already holds to the peer, + * so the probe would resolve without ever touching the address it is asking about. + */ + it('forces the dial so the address itself is contacted', async () => { + const forced: boolean[] = []; + const { network } = bareNetwork(); + (network as any).node.dial = async (_ma: unknown, opts?: { force?: boolean }): Promise => { + forced.push(opts?.force === true); + }; + await run(network); + expect(forced).toEqual([true]); + }); +}); /** * One peer can hold a configured address AND a gossip-learned one at the same time. * Deciding "is this configured" from the peer id let the gossip-learned sibling From 107a49129481deb6eb8cacf70df01185a5f7ba68 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 00:10:09 +0200 Subject: [PATCH 68/99] fix(network): honour the unreachable quarantine in redial maintenance --- backend/src/protocol/network.ts | 16 +++++- .../protocol/peer-eviction-guards.test.ts | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 42e65da8b..05848372f 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1020,6 +1020,7 @@ export class Network { let skippedBackoff = 0; let skippedNoReachable = 0; let skippedSuppressed = 0; + let skippedQuarantined = 0; const localCidrs = getLocalCidrs(now); for (const peer of allPeers) { if (epoch !== this.runEpoch) return; // stop() hit — this run's state is gone @@ -1037,6 +1038,17 @@ export class Network { skippedSuppressed++; continue; } + // A peer evicted as unreachable is normally gone from the peerStore and so + // cannot be a candidate at all — but that delete is best-effort, and mDNS, + // identify and peer-announce can all put the entry back. Honour the quarantine + // here too, or the very next tick dials the peer we just wrote off. Configured + // peers are never quarantined; the check is stated anyway so no future writer + // can hold user data back by adding one. + const quarantinedAt = this.unreachableQuarantine.get(pid); + if (quarantinedAt !== undefined && now - quarantinedAt < UNREACHABLE_QUARANTINE_MS && !this.configuredBootstrapPeerIDs.has(pid)) { + skippedQuarantined++; + continue; + } const bo = this.redialBackoff.get(pid); if (bo && bo.nextAttempt > now) { skippedBackoff++; @@ -1159,8 +1171,8 @@ export class Network { }; const workers = Array.from({ length: Math.min(CONCURRENCY, candidates.length) }, () => worker()); await Promise.all(workers); - if (candidates.length > 0 || skippedBackoff > 0 || skippedNoReachable > 0 || skippedSuppressed > 0) { - console.debug(` Re-dial: ${redialSuccess}/${candidates.length} succeeded (${skippedBackoff} skipped by backoff, ${skippedNoReachable} skipped no-reachable-addrs, ${skippedSuppressed} skipped left-peer)`); + if (candidates.length > 0 || skippedBackoff > 0 || skippedNoReachable > 0 || skippedSuppressed > 0 || skippedQuarantined > 0) { + console.debug(` Re-dial: ${redialSuccess}/${candidates.length} succeeded (${skippedBackoff} skipped by backoff, ${skippedNoReachable} skipped no-reachable-addrs, ${skippedSuppressed} skipped left-peer, ${skippedQuarantined} skipped quarantined)`); } // Prune backoff entries for peers no longer in peerStore to prevent unbounded growth. // Suppression is NOT pruned this way: leave-network purges the peer from the diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 73560474f..684a74af6 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1294,3 +1294,56 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => expect((network as any).redialBackoff.size).toBe(0); }); }); + +describe('runRedialMaintenance — quarantined peers are not candidates', () => { + function bareNetwork(quarantinedAt: number | null, configured = false) { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = quarantinedAt === null ? new Map() : new Map([[PEER_ID, quarantinedAt]]); + (network as any).noReachableSince = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { + getConnections: () => [], + async dial(id: { toString(): string }): Promise { + dialed.push(id.toString()); + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, dialed }; + } + + // A documentation-range public address, so the dial gater lets it through wherever + // this test happens to run. + const peer = { id: peerIdLike(PEER_ID), addresses: [{ multiaddr: multiaddr('/ip4/203.0.113.5/tcp/9090') }] }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [peer], 1); + + it('skips a peer still inside its unreachable quarantine', async () => { + const { network, dialed } = bareNetwork(Date.now() - 60_000); + await run(network); + expect(dialed).toEqual([]); + }); + + it('dials it again once the quarantine window has passed', async () => { + const { network, dialed } = bareNetwork(Date.now() - 10 * 60 * 60_000); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); + + it('never holds a configured peer back', async () => { + const { network, dialed } = bareNetwork(Date.now() - 60_000, true); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); + + it('dials a peer that was never quarantined', async () => { + const { network, dialed } = bareNetwork(null); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); +}); From 2ec86f5a781262cc3b5785b9bb86ad4a525b6f81 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 00:11:35 +0200 Subject: [PATCH 69/99] fix(network): classify an announced address by what it already is --- backend/src/protocol/network.ts | 37 +++++++++---- .../protocol/peer-eviction-guards.test.ts | 55 +++++++++++++++++++ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 05848372f..0c67e7d2b 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1526,6 +1526,19 @@ export class Network { // string: `/p2p//p2p-circuit/p2p/` contains our ID as the // relay hop yet targets a remote peer and must not be dropped as self. if (peerID === myPeerID) continue; + // What this address IS outranks what this caller calls it. The status tracker + // already keeps the stronger classification when a row is overwritten, so a + // gossip re-announcement of an address the user configured lands on a + // CONFIGURED row — and the dial has to follow the same rule or the two + // disagree. They did: the announce dialed without `force`, libp2p handed back + // the connection the peer happened to hold on a DIFFERENT address, and the + // discovered branch recorded 'connected' on a configured row whose address had + // never been contacted. The user then saw a green light on a broken entry. + const canonicalAddress = normalizeMultiaddrForCompare(ma.toString()); + const effectiveOrigin: BootstrapPeerOrigin = origin === 'configured' || this.configuredBootstrapAddresses.has(canonicalAddress) ? 'configured' : 'discovered'; + // Claiming the peer as configured stays keyed on what the CALLER declared: + // this branch also lifts leave-network suppression, which is the user's + // decision to reverse, never gossip's. if (peerID && origin === 'configured') { this.configuredBootstrapPeerIDs.add(peerID); // A re-configured bootstrap peer means its network was (re-)joined — it @@ -1546,7 +1559,7 @@ export class Network { // it here left every unreachable address a gossip flood could invent on the // list for good, since an ordinary timeout has nothing that takes it off. if (origin === 'configured') { - this.configuredBootstrapAddresses.add(normalizeMultiaddrForCompare(ma.toString())); + this.configuredBootstrapAddresses.add(canonicalAddress); this.rememberBootstrapAddress(ma); } // Safety net: refuse to dial loopback / unreachable-private bootstrap entries @@ -1556,14 +1569,14 @@ export class Network { // row instead: the user wrote it down and needs to see why nothing happens with it. if (shouldDenyDial(ma, localCidrs)) { trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); - if (origin === 'configured') this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'error', 'address is not routable from this host', null, origin); + if (effectiveOrigin === 'configured') this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'error', 'address is not routable from this host', null, effectiveOrigin); continue; } // Skip peers recently evicted as unreachable — nodes that still remember // them keep gossiping their addrs, and without this window every mention // would re-create the status row and burn a dial. Configured entries are // exempt: the user asked for them explicitly. - if (peerID && origin === 'discovered') { + if (peerID && effectiveOrigin === 'discovered') { const quarantinedAt = this.unreachableQuarantine.get(peerID); if (quarantinedAt !== undefined) { if (Date.now() - quarantinedAt < UNREACHABLE_QUARANTINE_MS) { @@ -1582,7 +1595,7 @@ export class Network { // peer from costing another dial, so it is claimed up front either way. if (peerID) this.bootstrapPeerIDs.add(peerID); console.debug('Adding bootstrap peer:', peer); - this.bootstrapTracker.markPending(networkID, peer, peerID, origin); + this.bootstrapTracker.markPending(networkID, peer, peerID, effectiveOrigin); try { // Always hand the address to libp2p and let IT decide whether a dial is // needed. Skipping the call whenever any connection to the peer existed @@ -1609,7 +1622,7 @@ export class Network { // that names many of them could otherwise make us open a connection per // address. For those, libp2p's own reuse is the desired behaviour. const pidObj = peerID ? peerIDFromString(peerID) : null; - const conn = await this.node.dial(ma, origin === 'configured' ? { force: true } : {}); + const conn = await this.node.dial(ma, effectiveOrigin === 'configured' ? { force: true } : {}); const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); // A dial already in flight cannot be called back: hangUp only closes // connections that ALREADY exist, so a leave-network landing mid-dial finds @@ -1641,15 +1654,15 @@ export class Network { // survive through another route is exactly what the row exists to expose. // Discovered rows carry the weaker "the peer answered" meaning and are // recorded either way. - if (origin === 'configured' && !verifiedThisAddr) { + if (effectiveOrigin === 'configured' && !verifiedThisAddr) { trace(`[NET] bootstrap addr unverified (connection came back on another address), left pending: ${peer}`); continue; } // A gossip-learned address has now answered on the endpoint it claimed, so // it has earned its place in the autodial list. Unverified ones never get // there, which is what keeps a flood of invented addresses off it. - if (origin === 'discovered' && verifiedThisAddr) this.rememberBootstrapAddress(ma); - this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); + if (effectiveOrigin === 'discovered' && verifiedThisAddr) this.rememberBootstrapAddress(ma); + this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, effectiveOrigin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { if (superseded()) return; @@ -1659,14 +1672,14 @@ export class Network { // again rather than letting the next announce buy another dial. if (probeAfterQuarantine && peerID) this.unreachableQuarantine.set(peerID, Date.now()); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; - this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, origin); + this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, effectiveOrigin); // [NET-MISMATCH] richer log for identity-mismatch — single line containing // origin (configured / discovered from peer-announce), multiaddr, // expected peerID and the actual peerID Noise reported. Makes it // trivial to grep `[NET-MISMATCH]` and diff what the catalog has // vs reality, even before the UI shows the same data. if (kind === 'identity-mismatch') { - console.log(`[NET-MISMATCH] origin=${origin} net=${networkID?.slice(0, 8) ?? 'none'} addr=${peer} expected=${peerID ?? 'none'} actual=${actualPeerID ?? 'unparsed'}`); + console.log(`[NET-MISMATCH] origin=${effectiveOrigin} net=${networkID?.slice(0, 8) ?? 'none'} addr=${peer} expected=${peerID ?? 'none'} actual=${actualPeerID ?? 'unparsed'}`); } else { console.log(`⚠️ Could not connect to bootstrap peer (${kind}): ${peer} — ${message}`); } @@ -1709,7 +1722,7 @@ export class Network { // Only once the peer has neither a live connection nor a single address we // have not disproved is there anything left to purge. if (this.node.getConnections(pid).length === 0 && remainingAddresses === 0) { - await this.purgeStalePeer(peerID, `${origin} dial identity mismatch, no usable address left`, epoch); + await this.purgeStalePeer(peerID, `${effectiveOrigin} dial identity mismatch, no usable address left`, epoch); } else { console.log(`[NET] dropped stale addr of peer ${peerID.slice(0, 16)}: ${ma.toString()}`); } @@ -1717,7 +1730,7 @@ export class Network { // status entry — there's no saved config row to "fix" and leaving // it visible just adds UI noise. For CONFIGURED entries, keep // it so the user can decide to update or remove the saved row. - if (origin === 'discovered' && networkID) { + if (effectiveOrigin === 'discovered' && networkID) { this.bootstrapTracker.deletePeer(networkID, peer); } } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 684a74af6..f5a3a312a 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1347,3 +1347,58 @@ describe('runRedialMaintenance — quarantined peers are not candidates', () => expect(dialed).toEqual([PEER_ID]); }); }); + +describe('addBootstrapPeers — a gossip announce of a configured address', () => { + const CONFIGURED_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const WORKING_B = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(knownConfigured: string[]) { + const outcomes: string[] = []; + const forced: boolean[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(knownConfigured.length > 0 ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(knownConfigured.map(a => normalizeMultiaddrForCompare(a))); + (network as any).unreachableQuarantine = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { + markPending() {}, + recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { + outcomes.push(status); + }, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [{}], // the peer was reachable via WORKING_B all along + async dial(_ma: unknown, opts?: { force?: boolean }): Promise { + forced.push(opts?.force === true); + // libp2p answers with the connection its OTHER address won. + return { remoteAddr: { toString: () => WORKING_B } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, outcomes, forced }; + } + + it('does not turn the configured row green off an unverified dial', async () => { + const { network, outcomes } = bareNetwork([CONFIGURED_A]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(outcomes).toEqual([]); + }); + + it('probes the address for real, as the configured branch would', async () => { + const { network, forced } = bareNetwork([CONFIGURED_A]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(forced).toEqual([true]); + }); + + it('still treats a genuinely unknown address as discovered', async () => { + const { network, outcomes, forced } = bareNetwork([]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(forced).toEqual([false]); + expect(outcomes).toEqual(['connected']); + }); +}); From 5d977b4e39a7bb738bb43518ff3dad1fcf465e46 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 00:15:03 +0200 Subject: [PATCH 70/99] fix(network): restore all purged state when a purge races a connection --- backend/src/protocol/network.ts | 71 ++++++++-- .../protocol/peer-eviction-guards.test.ts | 124 +++++++++++++++++- 2 files changed, 179 insertions(+), 16 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 0c67e7d2b..dabec9d7f 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1286,12 +1286,17 @@ export class Network { for (const ma of this.bootstrapMultiaddrs) { const pid = extractDestinationPeerID(ma); if (pid && this.isRedialSuppressed(pid)) continue; // deliberately left — don't resurrect it here - // A CONFIGURED entry is the user's way back in and is always tried; a - // DISCOVERED one earned its place here by answering once, but that is no - // reason to bypass the pacing re-dial maintenance applies to it. Without - // this, an isolated node re-dialed a dead discovered peer every 30s - // forever, since maintenance stops counting failures the moment we have no - // other connection to prove we are online. + // A CONFIGURED entry is the user's way back in, so it is never held back by the + // quarantine or by the per-peer eviction backoff — but it is still paced, on its + // own much shorter ADDRESS-level window. Without any pacing, several dead + // configured entries at a 10 s timeout each turn one tick into minutes of + // back-to-back dialing, every tick. + // + // A DISCOVERED entry earned its place here by answering once, which is no reason + // to bypass the pacing re-dial maintenance applies to it. Without that, an + // isolated node re-dialed a dead discovered peer every 30 s forever, since + // maintenance stops counting failures the moment we have no other connection to + // prove we are online. const canonical = normalizeMultiaddrForCompare(ma.toString()); const configured = this.configuredBootstrapAddresses.has(canonical); if (configured) { @@ -1816,22 +1821,60 @@ export class Network { // purge just removed — so if the peer is connected NOW, rebuild its dial // state from the live connections; otherwise reconnect would silently die // with the first drop. + // + // Not for a peer leave-network hung up: it is meant to be forgotten, and a + // connection racing the purge is not a reason to rebuild what the leave + // deliberately tore down. if (epoch !== this.runEpoch) return; const after = node.getConnections(pid); - if (after.length > 0) { - this.bootstrapPeerIDs.add(peerID); - this.unreachableQuarantine.delete(peerID); - await node.peerStore.merge(pid, { - multiaddrs: after.map(c => c.remoteAddr), - tags: { [KEEP_ALIVE]: { value: 1 } }, - }); - console.log(`[NET] purge raced an inbound connection — restored ${peerID.slice(0, 16)}…`); + if (after.length > 0 && !this.isRedialSuppressed(peerID)) { + await this.restorePurgedPeerState(node, pid, after, epoch); } } catch (err: any) { trace(`[NET] purgeStalePeer ${peerID.slice(0, 16)} failed: ${err?.message ?? err}`); } } + /** + * Put back everything {@link purgeStalePeer} took away, for a peer that turns out to + * be connected after all. + * + * The purge removes four things — the bootstrap dedup entry, the peer's addresses + * from the autodial list, its gossipsub direct entry and its keep-alive tag — and + * restoring only some of them left a state nothing else repairs: periodic promotion + * skips any peer already in `bootstrapPeerIDs`, so the missing address and direct + * entry would stay missing for as long as the peer stayed connected, and the next + * drop would find no way back. + * + * `bootstrapPeerIDs` is therefore filled in LAST. It is the flag the other paths read + * as "this peer is handled"; setting it first is what let promotion observe a + * half-restored peer and walk away from it. + */ + private async restorePurgedPeerState(node: Libp2p, pid: PeerID, connections: Array<{ remoteAddr: any }>, epoch: number): Promise { + const peerID = pid.toString(); + this.unreachableQuarantine.delete(peerID); + await node.peerStore.merge(pid, { + multiaddrs: connections.map(c => c.remoteAddr), + tags: { [KEEP_ALIVE]: { value: 1 } }, + }); + if (epoch !== this.runEpoch) return; + for (const c of connections) { + // The autodial list is walked by peer ID, so an address that does not already + // end in this peer's identity gets the suffix — the same shape promotion builds. + const remote = c.remoteAddr; + if (!remote) continue; + try { + this.rememberBootstrapAddress(extractDestinationPeerID(remote) === peerID ? remote : Multiaddr(`${remote.toString()}/p2p/${peerID}`)); + } catch { + // Unparseable remote address — nothing to put back on the list for it. + } + } + const gossipsub: any = this.pubsub; + if (gossipsub?.direct && typeof gossipsub.direct.add === 'function') gossipsub.direct.add(peerID); + this.bootstrapPeerIDs.add(peerID); + console.log(`[NET] purge raced an inbound connection — restored ${peerID.slice(0, 16)}…`); + } + /** * True if the peer is one we must never voluntarily disconnect because it * provides infrastructure rather than being a plain content peer: an diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index f5a3a312a..c9bdd0cbb 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -29,7 +29,6 @@ describe('purgeStalePeer — epoch guard', () => { (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).redialBackoff = new Map(); - (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).node = { getConnections: () => [ @@ -1009,6 +1008,7 @@ describe('probeParkedConfiguredBootstraps', () => { expect(forced).toEqual([true]); }); }); + /** * One peer can hold a configured address AND a gossip-learned one at the same time. * Deciding "is this configured" from the peer id let the gossip-learned sibling @@ -1135,6 +1135,11 @@ describe('Network.stop — per-run state really is per run', () => { }); }); +/** + * Zero-connection recovery used to work off the peer list the status tick snapshotted at + * its start — BEFORE re-dial maintenance ran. Maintenance reconnecting a peer in the + * meantime left recovery still believing it was isolated, so it dialed anyway. + */ describe('runZeroConnectionRecovery — connectivity is read, not remembered', () => { const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; @@ -1198,6 +1203,11 @@ describe('runZeroConnectionRecovery — connectivity is read, not remembered', ( }); }); +/** + * The recovery loop used to only LOG its failures. For an address whose peer is not in + * the peerStore, re-dial maintenance never sees the peer either — so nothing anywhere + * paced it and an isolated node re-dialed a dead entry on every 30 s tick, forever. + */ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => { const DISCOVERED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; const CONFIGURED = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; @@ -1269,7 +1279,6 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => * not unlimited: several dead ones at a 10 s timeout each turn every tick into * minutes of back-to-back dialing. */ - it('paces a configured address too, on its own record', async () => { const { network, dialed } = bareNetwork({ address: CONFIGURED, configured: true }); await run(network); @@ -1295,6 +1304,12 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => }); }); +/** + * An evicted peer is normally gone from the peerStore, so it cannot become a re-dial + * candidate at all — but that delete is best-effort and mDNS, identify and peer-announce + * can all put the entry straight back. Without a quarantine check here, the peer we just + * wrote off is dialed again on the very next tick. + */ describe('runRedialMaintenance — quarantined peers are not candidates', () => { function bareNetwork(quarantinedAt: number | null, configured = false) { const dialed: string[] = []; @@ -1348,6 +1363,13 @@ describe('runRedialMaintenance — quarantined peers are not candidates', () => }); }); +/** + * The status tracker keeps the STRONGER origin when a row is overwritten, so a gossip + * re-announcement of an address the user configured lands on a configured row. The dial + * followed the caller's origin instead: it went unforced, libp2p handed back the + * connection the peer held on a DIFFERENT address, and the discovered branch recorded + * 'connected' — a green light on a configured address that was never contacted. + */ describe('addBootstrapPeers — a gossip announce of a configured address', () => { const CONFIGURED_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; const WORKING_B = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; @@ -1402,3 +1424,101 @@ describe('addBootstrapPeers — a gossip announce of a configured address', () = expect(outcomes).toEqual(['connected']); }); }); + +/** + * purgeStalePeer takes four things away — the bootstrap dedup entry, the peer's + * addresses on the autodial list, its gossipsub direct entry and its keep-alive tag. + * The TOCTOU healing branch put only some of them back, and periodic promotion then + * skipped the peer precisely BECAUSE it was in bootstrapPeerIDs again, so the missing + * pieces were never filled in. + */ +describe('purgeStalePeer — healing an inbound race restores the whole dial state', () => { + const REMOTE = '/ip4/203.0.113.9/tcp/9090'; + + function bareNetwork(suppressed: string[] = []) { + const flagDuringDirectAdd: boolean[] = []; + const merges: Array> = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map([[PEER_ID, Date.now()]]); + (network as any).redialSuppressedByNet = new Map(suppressed.length > 0 ? [['net-a', new Set(suppressed)]] : []); + const direct = new Set(); + (network as any).pubsub = { + direct: { + add(id: string) { + // Ordering probe: bootstrapPeerIDs is the flag other paths read as + // "this peer is handled", so it must still be unset here. + flagDuringDirectAdd.push((network as any).bootstrapPeerIDs.has(PEER_ID)); + direct.add(id); + }, + delete: (id: string): boolean => direct.delete(id), + has: (id: string): boolean => direct.has(id), + }, + }; + (network as any).node = { + // The inbound connection that raced the purge is present throughout. + getConnections: () => [{ remoteAddr: multiaddr(REMOTE), async close(): Promise {} }], + peerStore: { + async delete(): Promise {}, + async merge(_pid: unknown, patch: Record): Promise { + merges.push(patch); + }, + }, + }; + return { network, direct, merges, flagDuringDirectAdd }; + } + + const run = (network: Network): Promise => (network as any).purgeStalePeer(PEER_ID, 'test', 1); + + it('puts the peer back in the bootstrap dedup set', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + it('puts its address back on the autodial list, carrying the peer identity', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([`${REMOTE}/p2p/${PEER_ID}`]); + }); + + it('puts it back in the gossipsub fast-reconnect set', async () => { + const { network, direct } = bareNetwork(); + await run(network); + expect(direct.has(PEER_ID)).toBe(true); + }); + + it('re-stamps the keep-alive tag', async () => { + const { network, merges } = bareNetwork(); + await run(network); + expect(merges).toHaveLength(1); + expect(merges[0]).toHaveProperty('tags'); + }); + + it('sets the bootstrap dedup flag last, so nothing can observe a half-healed peer', async () => { + const { network, flagDuringDirectAdd } = bareNetwork(); + await run(network); + expect(flagDuringDirectAdd).toEqual([false]); + }); + + it('lifts the unreachable quarantine', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); + + /** + * A peer hung up by leave-network is meant to be forgotten. A connection racing the + * purge is not a reason to rebuild the dial state the leave deliberately tore down. + */ + it('does not heal a peer the user left', async () => { + const { network, direct } = bareNetwork([PEER_ID]); + await run(network); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + expect(direct.has(PEER_ID)).toBe(false); + }); +}); From b54b73b3256323bfd8fe855fe72094a070713481 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 02:39:43 +0200 Subject: [PATCH 71/99] fix(peer-announce): require /p2p identity on discovered addrs --- backend/src/protocol/peer-announce.ts | 20 ++++++++-- .../tests/unit/protocol/peer-announce.test.ts | 38 ++++++++++++++++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index de0f6903f..773674eb3 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -1,7 +1,7 @@ import { trace } from '../logger.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; -import { canonicalMultiaddr } from './multiaddr-utils.ts'; +import { canonicalMultiaddr, extractDestinationPeerID } from './multiaddr-utils.ts'; import { LISH_TOPIC_PREFIX } from './constants.ts'; import { type Libp2p } from 'libp2p'; import { type BootstrapPeerOrigin } from '@shared'; @@ -270,14 +270,26 @@ export class PeerAnnounceManager { const unique = new Map(); let droppedNonRoutable = 0; let droppedDuplicate = 0; + let droppedAnonymous = 0; for (const a of data.multiaddrs) { if (typeof a !== 'string' || a.length === 0) continue; if (unique.size >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; try { - if (shouldDenyDial(Multiaddr(a), localCidrs)) { + const ma = Multiaddr(a); + if (shouldDenyDial(ma, localCidrs)) { droppedNonRoutable++; continue; } + // An address with no trailing /p2p/ names no identity, and every + // per-peer control downstream is keyed by one: backoff, quarantine, + // leave-network suppression and purge-by-peer-ID all silently do nothing + // for it, while a single successful dial is enough to park it on the + // recovery list that nothing can then take it off. Our own emitter always + // appends the identity, so requiring it costs nothing legitimate. + if (extractDestinationPeerID(ma) === null) { + droppedAnonymous++; + continue; + } } catch { // Unparseable multiaddr → drop (can't safely dial it anyway). droppedNonRoutable++; @@ -293,7 +305,7 @@ export class PeerAnnounceManager { unique.set(canonical, a); } if (unique.size === 0) { - if (droppedNonRoutable > 0 || droppedDuplicate > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${rawCount} addrs (${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate)`); + if (droppedNonRoutable > 0 || droppedDuplicate > 0 || droppedAnonymous > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${rawCount} addrs (${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, ${droppedAnonymous} without /p2p)`); return; } // Rate-limit AFTER dedup: a duplicate flood must not be able to drain the @@ -306,7 +318,7 @@ export class PeerAnnounceManager { } const filtered = admitted < unique.size ? [...unique.values()].slice(0, admitted) : [...unique.values()]; if (admitted < unique.size) trace(`[NET] peer-announce from ${source.slice(0, 16)}: rate limited to ${admitted}/${unique.size} addrs`); - trace(`[NET] peer-announce from ${source.slice(0, 16)}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, network ${networkID.slice(0, 8)})`); + trace(`[NET] peer-announce from ${source.slice(0, 16)}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, ${droppedAnonymous} without /p2p, network ${networkID.slice(0, 8)})`); // Pass networkID so per-peer outcomes from gossiped entries surface in the // UI under the network through which they arrived. Identity-mismatch // outcomes inside addBootstrapPeers also trigger purgeStalePeer. diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 4ccb8aa1b..991ca39f8 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -237,6 +237,13 @@ describe('PeerAnnounceManager.emit recently-seen membership', () => { const SRC_ID = '12D3KooWSourceSourceSourceSourceSourceSourceSourceSS'; const OTHER_SRC_ID = '12D3KooWOtherOtherOtherOtherOtherOtherOtherOtherOO'; +/** Identity every intake fixture terminates in — intake refuses addresses without one. */ +const ANNOUNCED_ID = '12D3KooWH3uVF6wv47WnArKHk5p6cvgCJEb74UTmxztmQDc298L3'; + +/** Append the announced identity, which inbound intake requires. */ +function withID(address: string, id: string = ANNOUNCED_ID): string { + return `${address}/p2p/${id}`; +} /** A manager wired only for handle(): captures the address lists it forwards. */ function intakeManager() { @@ -254,13 +261,13 @@ function intakeManager() { /** N distinct routable addresses in RFC5737 TEST-NET-3. */ function distinctAddrs(count: number): string[] { - return Array.from({ length: count }, (_v, i) => `/ip4/203.0.113.${i % 254}/tcp/${9000 + i}`); + return Array.from({ length: count }, (_v, i) => withID(`/ip4/203.0.113.${i % 254}/tcp/${9000 + i}`)); } describe('PeerAnnounceManager.handle address dedup', () => { it('collapses one address repeated many times into a single entry', async () => { const { mgr, forwarded } = intakeManager(); - const addr = '/ip4/198.51.100.7/tcp/9090'; + const addr = withID('/ip4/198.51.100.7/tcp/9090'); await mgr.handle({ type: 'peer-announce', multiaddrs: Array(300).fill(addr) }, 'netAAAA', SRC_ID); @@ -269,7 +276,7 @@ describe('PeerAnnounceManager.handle address dedup', () => { it('collapses two spellings of one address (DNS case, expanded vs compressed IPv6)', async () => { const { mgr, forwarded } = intakeManager(); - const multiaddrs = [`/dns4/Peer.Example.COM/tcp/9090/p2p/${SELF_ID}`, `/dns4/peer.example.com/tcp/9090/p2p/${SELF_ID}`, '/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090', '/ip6/2001:db8::1/tcp/9090']; + const multiaddrs = [`/dns4/Peer.Example.COM/tcp/9090/p2p/${SELF_ID}`, `/dns4/peer.example.com/tcp/9090/p2p/${SELF_ID}`, withID('/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090'), withID('/ip6/2001:db8::1/tcp/9090')]; await mgr.handle({ type: 'peer-announce', multiaddrs }, 'netAAAA', SRC_ID); @@ -281,7 +288,7 @@ describe('PeerAnnounceManager.handle address dedup', () => { // 300 copies of one address plus 5 distinct ones is 6 unique — the duplicates // must not consume the 128-address budget the distinct ones need. const { mgr, forwarded } = intakeManager(); - const dup = '/ip4/198.51.100.7/tcp/9090'; + const dup = withID('/ip4/198.51.100.7/tcp/9090'); const rest = distinctAddrs(5); await mgr.handle({ type: 'peer-announce', multiaddrs: [...Array(300).fill(dup), ...rest] }, 'netAAAA', SRC_ID); @@ -300,10 +307,31 @@ describe('PeerAnnounceManager.handle address dedup', () => { it('drops non-routable addresses before deduping', async () => { const { mgr, forwarded } = intakeManager(); - await mgr.handle({ type: 'peer-announce', multiaddrs: ['/ip4/127.0.0.1/tcp/9090', '/ip4/127.0.0.1/tcp/9090', 'not-a-multiaddr'] }, 'netAAAA', SRC_ID); + await mgr.handle({ type: 'peer-announce', multiaddrs: [withID('/ip4/127.0.0.1/tcp/9090'), withID('/ip4/127.0.0.1/tcp/9090'), 'not-a-multiaddr'] }, 'netAAAA', SRC_ID); expect(forwarded).toEqual([]); }); + + it('refuses an announced address that carries no /p2p identity', async () => { + // Without an identity the address is unreachable by every per-peer control + // downstream — backoff, quarantine, leave-suppression, purge — while one + // successful dial parks it on the recovery list for good. + const { mgr, forwarded } = intakeManager(); + const named = withID('/ip4/203.0.113.5/tcp/9090'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: ['/ip4/203.0.113.4/tcp/9090', named] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[named]]); + }); + + it('keeps the relay-circuit target identity, not the relay hop', async () => { + const { mgr, forwarded } = intakeManager(); + const relayed = `/ip4/203.0.113.6/tcp/9090/p2p/${SELF_ID}/p2p-circuit/p2p/${ANNOUNCED_ID}`; + + await mgr.handle({ type: 'peer-announce', multiaddrs: [relayed] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[relayed]]); + }); }); describe('PeerAnnounceManager.handle per-source rate limit', () => { From 7415d4dc2bb3730bfec90bab561242e3236fb448 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 02:45:02 +0200 Subject: [PATCH 72/99] fix(network): pace discovered bootstrap dials with the peer backoff --- backend/src/protocol/network.ts | 20 +++++ .../unit/protocol/network-disconnect.test.ts | 3 + .../protocol/peer-eviction-guards.test.ts | 87 +++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index dabec9d7f..524503939 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1577,6 +1577,19 @@ export class Network { if (effectiveOrigin === 'configured') this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'error', 'address is not routable from this host', null, effectiveOrigin); continue; } + // Pacing, before the quarantine is consulted: gossip mentions a dead peer + // on every announce cycle, and this path used to answer each one with a + // fresh 10 s dial because it read the quarantine and nothing else. The + // backoff is the record every other dial path already waits on, and + // checking it FIRST matters — an expired quarantine buys exactly one probe, + // which must not be spent by a dial the backoff was going to refuse. + if (peerID && effectiveOrigin === 'discovered') { + const backoff = this.redialBackoff.get(peerID); + if (backoff !== undefined && backoff.nextAttempt > Date.now()) { + trace(`[NET] addBootstrapPeers skip backoff: ${peerID.slice(0, 16)}`); + continue; + } + } // Skip peers recently evicted as unreachable — nodes that still remember // them keep gossiping their addrs, and without this window every mention // would re-create the status row and burn a dial. Configured entries are @@ -1676,6 +1689,13 @@ export class Network { // The probe the expired quarantine allowed has failed, so close the window // again rather than letting the next announce buy another dial. if (probeAfterQuarantine && peerID) this.unreachableQuarantine.set(peerID, Date.now()); + // Pay the failure into the shared per-peer backoff the check above reads. + // Without this the check could never bite for a gossip-learned peer: nothing + // else writes that record for a peer absent from the peerStore, so re-dial + // maintenance never sees it either and every announce bought another dial. + // Same accounting as the recovery loop — pacing only, no eviction credit, + // since a failed announce dial says nothing about who is the broken side. + if (peerID && effectiveOrigin === 'discovered') this.noteRecoveryDialFailure(peerID); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, effectiveOrigin); // [NET-MISMATCH] richer log for identity-mismatch — single line containing diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index 9a4cc0a34..22528be3e 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -125,6 +125,7 @@ describe('Network.runRedialMaintenance — leave-peer suppression', () => { (network as any).redialBackoff = new Map(); (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).noReachableSince = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); @@ -184,6 +185,7 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { (network as any).redialBackoff = new Map(); (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapMultiaddrs = bootstrapMaStrs.map(s => multiaddr(s)); @@ -260,6 +262,7 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapGeneration = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index c9bdd0cbb..485945723 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -30,6 +30,7 @@ describe('purgeStalePeer — epoch guard', () => { (network as any).bootstrapMultiaddrs = []; (network as any).redialBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).node = { getConnections: () => [ { @@ -92,6 +93,7 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { (network as any).redialBackoff = new Map(); (network as any).redialSuppressedByNet = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); (network as any).configuredBootstrapAddresses = new Set(); @@ -162,6 +164,7 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -325,6 +328,7 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -370,6 +374,7 @@ describe('configured exemption ends when the peer leaves the config', () => { (network as any).redialBackoff = new Map(); (network as any).redialSuppressedByNet = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - 45 * 60_000]]); (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); (network as any).configuredBootstrapAddresses = new Set(); @@ -457,6 +462,7 @@ describe('addBootstrapPeers — superseded bootstrap configuration', () => { (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -519,6 +525,7 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -612,6 +619,7 @@ describe('addBootstrapPeers — only a working discovered address joins the auto (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -699,6 +707,7 @@ describe('addBootstrapPeers — a non-routable configured entry is still configu (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -771,6 +780,7 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map([[PEER_ID, quarantinedAt]]); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -805,6 +815,7 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { it('does not quarantine a plain failure that was never in one', async () => { const network = bareNetwork(true, LONG_AGO); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); }); @@ -855,6 +866,7 @@ describe('addBootstrapPeers — identity mismatch trims the address, not the pee (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; (network as any).bootstrapGeneration = new Map(); @@ -1157,6 +1169,7 @@ describe('runZeroConnectionRecovery — connectivity is read, not remembered', ( (network as any).redialBackoff = new Map(); (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(); (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapMultiaddrs = (opts.addresses ?? [ADDR_A]).map(a => multiaddr(a)); @@ -1220,6 +1233,7 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => (network as any).redialBackoff = new Map(); (network as any).addressProbeBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); (network as any).configuredBootstrapAddresses = new Set(opts.configured ? [normalizeMultiaddrForCompare(opts.address)] : []); (network as any).bootstrapMultiaddrs = [multiaddr(opts.address)]; @@ -1383,6 +1397,7 @@ describe('addBootstrapPeers — a gossip announce of a configured address', () = (network as any).configuredBootstrapPeerIDs = new Set(knownConfigured.length > 0 ? [PEER_ID] : []); (network as any).configuredBootstrapAddresses = new Set(knownConfigured.map(a => normalizeMultiaddrForCompare(a))); (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); @@ -1522,3 +1537,75 @@ describe('purgeStalePeer — healing an inbound race restores the whole dial sta expect(direct.has(PEER_ID)).toBe(false); }); }); + +/** + * Gossip re-announces a dead peer on every cycle. The intake path used to answer each + * mention with a fresh dial because it consulted the unreachable quarantine and nothing + * else — the per-peer backoff that paces every other dial path was never read, and never + * written either, so it could not have bitten even if it had been. + */ +describe('addBootstrapPeers — discovered dials are paced by the per-peer backoff', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(dialOutcome: 'ok' | 'fail') { + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + if (dialOutcome === 'fail') throw new Error('dial timed out'); + return { remoteAddr: { toString: () => ADDR } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, dialled }; + } + + it('records a failed discovered dial into the shared backoff', async () => { + const { network } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).redialBackoff.get(PEER_ID)?.nextAttempt).toBeGreaterThan(Date.now()); + }); + + it('refuses a second mention of the same peer while the backoff is running', async () => { + const { network, dialled } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(dialled).toEqual([ADDR]); + }); + + it('dials again once the backoff window has passed', async () => { + const { network, dialled } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() - 1, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(dialled).toEqual([ADDR, ADDR]); + }); + + /** A configured entry is user data and the way back in — the backoff must not hold it. */ + it('never holds a configured address back on the peer backoff', async () => { + const { network, dialled } = bareNetwork('fail'); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() + 600_000, failCount: 9, firstFailure: Date.now(), evictionFails: 0 }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR]); + }); + + /** A dial that worked clears the record, so a returning peer is not paced. */ + it('leaves no backoff behind after a successful discovered dial', async () => { + const { network } = bareNetwork('ok'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).redialBackoff.has(PEER_ID)).toBe(false); + }); +}); From b691dc46b2858f834198ef59db2e3971954f9e9d Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 02:48:03 +0200 Subject: [PATCH 73/99] fix(bootstrap-status): age discovered rows from last success --- backend/src/protocol/bootstrap-status.ts | 46 +++++++++++++------ .../unit/protocol/bootstrap-status.test.ts | 31 +++++++++++-- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 94fe52081..889a51e52 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -9,6 +9,19 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialS */ const MAX_DISCOVERED_PER_NETWORK = 256; +/** + * A stored row: the status the UI sees, plus the clock {@link BootstrapStatusTracker.sweepStale} + * measures against. + * + * `updatedAt` cannot serve as that clock. It is "when did anything about this row last + * change", which is what the UI wants, and it moves on every dial outcome — failures + * included. Gossip re-announces a dead peer far more often than the sweep TTL, and this + * node answers each mention with a dial that fails, so measuring staleness from + * `updatedAt` meant a dead row was refreshed by its own failures and could never expire. + * `staleSince` moves only when the address actually answered. + */ +type TrackedPeer = BootstrapPeerStatus & { staleSince: number }; + /** * Tracks per-network, per-bootstrap-peer dial outcome status. * @@ -24,7 +37,7 @@ const MAX_DISCOVERED_PER_NETWORK = 256; * under the network through which they were learned. */ export class BootstrapStatusTracker { - private readonly stats: Map> = new Map(); + private readonly stats: Map> = new Map(); private onStatusChange: ((networkID: string, status: BootstrapStatus) => void) | null = null; /** Open {@link batch} frames, keyed by networkID. See that method for why. */ private readonly batches: Map = new Map(); @@ -109,7 +122,7 @@ export class BootstrapStatusTracker { } /** Iterate over all tracked network IDs and their peer maps. Used for NET-CHURN dump. */ - entries(): IterableIterator<[string, Map]> { + entries(): IterableIterator<[string, Map]> { return this.stats.entries(); } @@ -138,8 +151,8 @@ export class BootstrapStatusTracker { // a mention move the clock made a dead peer's row immortal: every announce cycle // is far shorter than the sweep TTL, so the row was refreshed long before it // could expire, no matter how many dials to it had already failed. Only a dial - // that actually produced an outcome advances it, in recordOutcome below. - net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString() }); + // that actually CONNECTED advances it, in recordOutcome below. + net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString(), staleSince: previous?.staleSince ?? Date.now() }); this.capDiscovered(net); this.notify(networkID); } @@ -151,13 +164,17 @@ export class BootstrapStatusTracker { const truncated = message ? (message.length > 200 ? message.slice(0, 200) + '…' : message) : null; const previous = net.get(multiaddr); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; - net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString() }); + // Only success restarts the staleness clock. A FAILING outcome is the node's own + // reaction to somebody else's mention of a dead peer, so letting it advance the + // clock kept exactly the rows this sweep exists to remove: gossip mentions the + // peer, the dial fails, the row is refreshed, and the TTL is never reached. + net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString(), staleSince: status === 'connected' ? Date.now() : (previous?.staleSince ?? Date.now()) }); this.capDiscovered(net); this.notify(networkID); } /** Bound discovered rows per network (drop the oldest) — see MAX_DISCOVERED_PER_NETWORK. */ - private capDiscovered(net: Map): void { + private capDiscovered(net: Map): void { let discovered = 0; for (const p of net.values()) if (p.origin === 'discovered') discovered++; if (discovered <= MAX_DISCOVERED_PER_NETWORK) return; @@ -196,10 +213,11 @@ export class BootstrapStatusTracker { } /** - * Drop discovered-origin entries that have gone stale: no status refresh within - * `ttlMs` AND the peer is not an active member of THAT network. Dead peers stop - * being mentioned by gossip, so their rows stop refreshing and expire here — - * including rows frozen at 'connected' for a peer that silently died. The + * Drop discovered-origin entries that have gone stale: nothing has CONNECTED on the + * address within `ttlMs` AND the peer is not an active member of THAT network. A peer + * that dies stops answering, so its clock freezes and the row expires here — whether + * gossip keeps naming it or not, and whether the row is frozen at 'connected' or + * cycling through failures that this node produces itself. See {@link TrackedPeer}. The * liveness predicate is scoped to the network (its topic subscribers), NOT the * shared libp2p connection: a peer that left network B but is still connected * through network A must not keep a stale row under B. Configured entries are @@ -212,8 +230,7 @@ export class BootstrapStatusTracker { if (p.origin !== 'discovered') continue; const pid = p.expectedPeerID ?? p.actualPeerID; if (pid && isMember(networkID, pid)) continue; - const updated = Date.parse(p.updatedAt); - if (Number.isFinite(updated) && now - updated < ttlMs) continue; + if (now - p.staleSince < ttlMs) continue; peers.delete(addr); changed = true; } @@ -262,7 +279,7 @@ export class BootstrapStatusTracker { this.batches.clear(); } - private ensureNetwork(networkID: string): Map { + private ensureNetwork(networkID: string): Map { let net = this.stats.get(networkID); if (!net) { net = new Map(); @@ -274,6 +291,7 @@ export class BootstrapStatusTracker { private buildStatus(networkID: string): BootstrapStatus | null { const peers = this.stats.get(networkID); if (!peers) return null; - return { networkID, peers: [...peers.values()].map(p => ({ ...p })) }; + // staleSince is internal bookkeeping, not part of the wire contract. + return { networkID, peers: [...peers.values()].map(({ staleSince: _staleSince, ...p }) => p) }; } } diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 7542613c9..a7c3066b3 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -208,7 +208,12 @@ describe('BootstrapStatusTracker.sweepStale', () => { expect(tracker.getStatus(NET)).toBe(null); // ages out from the last real outcome }); - it('lets a real dial outcome refresh the clock', async () => { + /** + * The failure is this node's own reaction to somebody else's mention of a dead peer. + * Counting it as activity was the same immortality bug as the mention itself: gossip + * names the peer, the dial fails, the row is refreshed, and the TTL never arrives. + */ + it('does not let a failed dial outcome refresh the staleness clock', async () => { const tracker = new BootstrapStatusTracker(); tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); const firstAt = clockOf(tracker); @@ -216,9 +221,29 @@ describe('BootstrapStatusTracker.sweepStale', () => { tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); - expect(Date.parse(clockOf(tracker))).toBeGreaterThan(Date.parse(firstAt)); + expect(Date.parse(clockOf(tracker))).toBeGreaterThan(Date.parse(firstAt)); // display clock moves tracker.sweepStale(TTL, () => false, Date.parse(firstAt) + TTL + 2); - expect(tracker.getStatus(NET)?.peers.length).toBe(1); // survives — clock moved + expect(tracker.getStatus(NET)).toBe(null); // staleness clock did not + }); + + it('lets a successful dial refresh the staleness clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const firstAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, () => false, Date.parse(firstAt) + TTL + 2); + expect(tracker.getStatus(NET)?.peers.length).toBe(1); // survives — the address answered + }); + + /** The clock is bookkeeping, not part of what the API hands out. */ + it('keeps the staleness clock out of the published snapshot', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + expect(tracker.getStatus(NET)!.peers[0]!).not.toHaveProperty('staleSince'); }); it('starts the clock on a first mention that has no prior row', () => { From 98aca8d6fffbd769c6b39e97e7a18ee59095b986 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 02:56:55 +0200 Subject: [PATCH 74/99] fix(network): single-flight bootstrap dials per address --- backend/src/protocol/network.ts | 29 +++++ .../unit/protocol/network-disconnect.test.ts | 2 + .../protocol/peer-eviction-guards.test.ts | 101 +++++++++++++++++- 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 524503939..c31c1ab29 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -368,6 +368,17 @@ export class Network { * starts at the dead one again — so the working address could go untried indefinitely. */ private readonly addressProbeBackoff = new Map(); + /** + * Canonical multiaddrs with an {@link addBootstrapPeers} dial currently outstanding. + * + * The pubsub dispatcher does not await the announce handler, so several announces + * naming the same address start several intake runs that overlap. Each spends its own + * 10 s dial timeout on the same endpoint and records its own outcome over the other's, + * and the peer-level backoff cannot help — it is only written once a dial has already + * failed. Claiming the address for the duration of the dial makes the second run a + * no-op instead: the first one is about to record the outcome both were after. + */ + private readonly inFlightBootstrapDials = new Set(); /** * peerID → time we first saw the peer disconnected with ZERO reachable * addresses. Such peers never enter the re-dial path (nothing to dial), so @@ -1609,6 +1620,16 @@ export class Network { probeAfterQuarantine = true; } } + // Single-flight, keyed by the endpoint rather than the peer: two addresses of + // one peer are two different questions and both deserve their own dial, while + // two runs asking about the SAME address duplicate a 10 s timeout for one + // answer. Claimed after every skip above so a refused candidate never blocks + // the run that would actually dial it. + if (this.inFlightBootstrapDials.has(canonicalAddress)) { + trace(`[NET] addBootstrapPeers skip in-flight: ${peer}`); + continue; + } + this.inFlightBootstrapDials.add(canonicalAddress); // The identity set is the dedup that stops every gossip mention of the same // peer from costing another dial, so it is claimed up front either way. if (peerID) this.bootstrapPeerIDs.add(peerID); @@ -1759,6 +1780,11 @@ export class Network { this.bootstrapTracker.deletePeer(networkID, peer); } } + } finally { + // Every exit from the dial block releases the claim, `return` included — + // a leave landing mid-dial would otherwise lock the address out for the + // lifetime of the node. + this.inFlightBootstrapDials.delete(canonicalAddress); } } catch (error: any) { this.bootstrapTracker.recordOutcome(networkID, peer, null, 'error', error?.message ?? String(error), null, origin); @@ -2598,6 +2624,9 @@ export class Network { this.bootstrapTracker.clear(); this.bootstrapMultiaddrs = []; this.bootstrapGeneration.clear(); + // Claims belong to the node being torn down; a dial still settling on the old node + // must not lock the address out for the next one. + this.inFlightBootstrapDials.clear(); this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index 22528be3e..25df7c5d8 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -20,6 +20,7 @@ function makeNetwork() { const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map>(); (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).redialBackoff = new Map(); @@ -264,6 +265,7 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { (network as any).unreachableQuarantine = new Map(); (network as any).redialBackoff = new Map(); (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 485945723..f333ecda8 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -168,6 +168,7 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); const outcomes: string[] = []; (network as any).bootstrapTracker = { markPending() {}, @@ -332,6 +333,7 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -466,6 +468,7 @@ describe('addBootstrapPeers — superseded bootstrap configuration', () => { (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -529,6 +532,7 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).disconnectPeer = async (peerID: string): Promise => { disconnected.push(peerID); @@ -623,6 +627,7 @@ describe('addBootstrapPeers — only a working discovered address joins the auto (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -711,6 +716,7 @@ describe('addBootstrapPeers — a non-routable configured entry is still configu (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome(_n: unknown, _a: unknown, _p: unknown, status: string, message: string | null) { @@ -784,6 +790,7 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -870,6 +877,7 @@ describe('addBootstrapPeers — identity mismatch trims the address, not the pee (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; (network as any).purgeStalePeer = async (id: string): Promise => { purged.push(id); @@ -1093,8 +1101,8 @@ describe('configured origin is a property of the address, not the peer', () => { describe('Network.stop — per-run state really is per run', () => { function bareNetwork() { const network = Object.create(Network.prototype) as Network; - for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'addressProbeBackoff', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys']) { - (network as any)[field] = field === 'seenSearchIDs' || field === 'dcutrPeers' || field === 'bootstrapPeerIDs' || field === 'configuredBootstrapPeerIDs' || field === 'configuredBootstrapAddresses' ? new Set() : new Map(); + for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'addressProbeBackoff', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys', 'inFlightBootstrapDials']) { + (network as any)[field] = field === 'seenSearchIDs' || field === 'dcutrPeers' || field === 'bootstrapPeerIDs' || field === 'configuredBootstrapPeerIDs' || field === 'configuredBootstrapAddresses' || field === 'inFlightBootstrapDials' ? new Set() : new Map(); } (network as any).runEpoch = 1; (network as any).statusInterval = null; @@ -1401,6 +1409,7 @@ describe('addBootstrapPeers — a gossip announce of a configured address', () = (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { @@ -1559,6 +1568,7 @@ describe('addBootstrapPeers — discovered dials are paced by the per-peer backo (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; (network as any).node = { peerId: { toString: () => 'selfID' }, @@ -1609,3 +1619,90 @@ describe('addBootstrapPeers — discovered dials are paced by the per-peer backo expect((network as any).redialBackoff.has(PEER_ID)).toBe(false); }); }); + +/** + * The pubsub dispatcher does not await the announce handler, so two announces naming the + * same address run their intake concurrently. Each used to spend its own 10 s dial + * timeout on one endpoint, and the peer backoff cannot help — it is only written once a + * dial has already failed. + */ +describe('addBootstrapPeers — one dial per address at a time', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const OTHER_ADDR = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const dialled: string[] = []; + // Every dial parks here until released, so a second intake run genuinely overlaps + // the first instead of merely following it. + const pending: Array<() => void> = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + await new Promise(resolve => pending.push(resolve)); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { + network, + dialled, + releaseDials: (): void => { + for (const resolve of pending.splice(0)) resolve(); + }, + }; + } + + it('drops a second intake run while the first is still dialing the address', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); // let the first run reach its dial + // Not awaited: without the claim the second run parks on its own dial, and awaiting + // it here would hang the test instead of failing the assertion below. + const second = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + + expect(dialled).toEqual([ADDR]); + releaseDials(); + await Promise.all([first, second]); + }); + + it('still dials a different address of the same peer concurrently', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + const second = (network as any).addBootstrapPeers([OTHER_ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + + expect(dialled).toEqual([ADDR, OTHER_ADDR]); + releaseDials(); + await Promise.all([first, second]); + }); + + it('releases the claim once the dial settles, so a later mention can dial again', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + releaseDials(); + await first; + const second = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + releaseDials(); + await second; + + expect(dialled).toEqual([ADDR, ADDR]); + }); +}); From 5d535514bded27b4234e723f867b6f7ad619deaf Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 02:59:10 +0200 Subject: [PATCH 75/99] fix(network): gate peer:discovery dials on quarantine and backoff --- backend/src/protocol/network.ts | 45 +++++-- .../protocol/peer-discovery-handler.test.ts | 120 ++++++++++++++++++ 2 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 backend/tests/unit/protocol/peer-discovery-handler.test.ts diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index c31c1ab29..838f01a65 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -731,28 +731,55 @@ export class Network { // re-dialed by discovery (mDNS/identify/PX) — that would beat the disconnect. // Suppression lifts on a legitimate inbound reconnect or on network rejoin. if (this.isRedialSuppressed(peerID)) return; - // Stamp `keep-alive-fleet` on every discovered peer, regardless of how they + // Stamp `keep-alive-fleet` on a peer we are actually CONNECTED to, however it // surfaced (mDNS, bootstrap, autonat, identify, peer-announce). libp2p // ReconnectQueue only acts on peers with a tag whose key starts with // `keep-alive`; without it, fleet peers found via non-announce channels // (e.g. identify push from a common neighbour) are not re-dialed when // they drop. Value 50 sits between bootstrap (100) and idle (1) — protects // from ConnectionPruner without taking precedence over true bootstraps. - try { - await this.node!.peerStore.merge(evt.detail.id, { - tags: { 'keep-alive-fleet': { value: 50 } }, - }); - } catch { - /* ignore */ - } + // + // Never on a mere discovery event, though: the tag is a standing instruction + // to libp2p to keep re-dialing this identity, and a discovery event is only + // somebody's claim that the peer exists. Stamping it before any contact let a + // peer that had just been evicted as unreachable get its re-dial instruction + // back from a late mDNS or PX event, ReconnectQueue included. + const tagAsFleetPeer = async (): Promise => { + try { + await this.node!.peerStore.merge(evt.detail.id, { + tags: { 'keep-alive-fleet': { value: 50 } }, + }); + } catch { + /* ignore */ + } + }; const existing = this.node!.getConnections(evt.detail.id); - if (existing.length > 0) return; + if (existing.length > 0) { + await tagAsFleetPeer(); + return; + } if (!evt.detail.multiaddrs?.length) return; + // The same pacing every other dial path respects. Discovery is a firehose — + // mDNS, identify and PX all deliver events for peers we have already written + // off — and this handler used to answer each one with an immediate dial, which + // could undo an eviction the moment it happened. + if (!isRecoveryDialDue(peerID, Date.now(), this.redialBackoff, this.unreachableQuarantine)) { + trace(`[NET] discovery dial skipped (quarantined or in backoff): ${peerID.slice(0, 16)}`); + return; + } + const epoch = this.runEpoch; try { await this.node!.dial(evt.detail.multiaddrs); + // A dial that settles after stop() belongs to a node this run no longer owns. + if (epoch !== this.runEpoch) return; + await tagAsFleetPeer(); trace(`[NET] Dialed discovered peer ${peerID.slice(0, 16)}`); } catch (err: any) { + if (epoch !== this.runEpoch) return; + // Pay the failure into the shared backoff the gate above reads, so a peer + // discovery keeps naming is paced like everything else. + this.noteRecoveryDialFailure(peerID); trace(`[NET] Failed to dial discovered peer ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } }); diff --git a/backend/tests/unit/protocol/peer-discovery-handler.test.ts b/backend/tests/unit/protocol/peer-discovery-handler.test.ts new file mode 100644 index 000000000..fe30046a9 --- /dev/null +++ b/backend/tests/unit/protocol/peer-discovery-handler.test.ts @@ -0,0 +1,120 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * The `peer:discovery` handler. libp2p delivers these events from mDNS, identify and + * gossipsub PX alike, including for peers this node has already written off — so the + * handler is a dial path like any other and has to respect the same pacing. It used to + * respect only leave-network suppression, and it stamped the `keep-alive-fleet` re-dial + * instruction before making any contact at all. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const ADDR = { toString: () => `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}` }; + +type Handler = (evt: any) => void | Promise; + +function bareNetwork(opts: { connected?: boolean; dialFails?: boolean } = {}) { + const dialled: unknown[][] = []; + const tagged: string[] = []; + const handlers = new Map(); + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).listeners = []; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).dcutrPeers = new Set(); + (network as any).peerDisconnectHandlers = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).recentDisconnects = []; + (network as any).lastMeshChange = new Map(); + (network as any).pubsub = null; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + addEventListener(event: string, handler: Handler) { + handlers.set(event, handler); + }, + getConnections: () => (opts.connected ? [{}] : []), + getPeers: () => [], + async dial(multiaddrs: unknown[]): Promise { + dialled.push(multiaddrs); + if (opts.dialFails) throw new Error('dial timed out'); + return {}; + }, + peerStore: { + async merge(_pid: unknown, patch: { tags?: Record }): Promise { + for (const key of Object.keys(patch.tags ?? {})) tagged.push(key); + }, + }, + }; + (network as any).setupEventListeners(); + const fire = async (): Promise => { + await handlers.get('peer:discovery')!({ detail: { id: { toString: () => PEER_ID }, multiaddrs: [ADDR] } }); + }; + return { network, dialled, tagged, fire }; +} + +describe('peer:discovery — pacing', () => { + it('dials a peer nothing is holding back', async () => { + const { dialled, fire } = bareNetwork(); + await fire(); + expect(dialled).toHaveLength(1); + }); + + /** + * The eviction path quarantines a peer precisely so it stops being re-dialed. A late + * discovery event arriving right after must not undo that. + */ + it('refuses to dial a quarantined peer', async () => { + const { network, dialled, fire } = bareNetwork(); + (network as any).unreachableQuarantine.set(PEER_ID, Date.now()); + await fire(); + expect(dialled).toEqual([]); + }); + + it('refuses to dial a peer inside its redial backoff', async () => { + const { network, dialled, fire } = bareNetwork(); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 2, firstFailure: Date.now(), evictionFails: 0 }); + await fire(); + expect(dialled).toEqual([]); + }); + + it('records a failed discovery dial into the backoff', async () => { + const { network, fire } = bareNetwork({ dialFails: true }); + await fire(); + expect((network as any).redialBackoff.get(PEER_ID)?.nextAttempt).toBeGreaterThan(Date.now()); + }); +}); + +describe('peer:discovery — keep-alive tagging', () => { + /** + * The tag is a standing instruction to libp2p's ReconnectQueue. Writing it off an + * unverified claim is what let an evicted peer get its re-dial instruction back. + */ + it('does not tag a peer whose dial failed', async () => { + const { tagged, fire } = bareNetwork({ dialFails: true }); + await fire(); + expect(tagged).toEqual([]); + }); + + it('does not tag a quarantined peer it never contacted', async () => { + const { network, tagged, fire } = bareNetwork(); + (network as any).unreachableQuarantine.set(PEER_ID, Date.now()); + await fire(); + expect(tagged).toEqual([]); + }); + + it('tags a peer once the dial succeeds', async () => { + const { tagged, fire } = bareNetwork(); + await fire(); + expect(tagged).toEqual(['keep-alive-fleet']); + }); + + it('tags a peer we are already connected to, without dialing again', async () => { + const { dialled, tagged, fire } = bareNetwork({ connected: true }); + await fire(); + expect(tagged).toEqual(['keep-alive-fleet']); + expect(dialled).toEqual([]); + }); +}); From 597c98e4c64f54ea2c37fcd6706111102fc2a4cd Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:03:26 +0200 Subject: [PATCH 76/99] fix(network): close a discovery dial that races leave-network --- backend/src/protocol/network.ts | 24 +++- .../protocol/peer-discovery-handler.test.ts | 116 ++++++++++++++++++ 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 838f01a65..cf6646c58 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -773,6 +773,19 @@ export class Network { await this.node!.dial(evt.detail.multiaddrs); // A dial that settles after stop() belongs to a node this run no longer owns. if (epoch !== this.runEpoch) return; + // The suppression check at entry answered for the moment the event arrived, + // and a dial takes seconds. leave-network can land inside that window: its + // hangUp finds no connection yet, finishes, and this dial then completes into + // a connection nothing else will close. Whoever notices last closes it. + if (this.isRedialSuppressed(peerID) && !this.isPeerNeededByJoinedNetwork(peerID)) { + trace(`[NET] discovery dial landed after leave, hanging up: ${peerID.slice(0, 16)}`); + try { + await this.node!.hangUp(evt.detail.id); + } catch (err: any) { + trace(`[NET] hangUp of late discovery dial failed: ${err?.message ?? err}`); + } + return; + } await tagAsFleetPeer(); trace(`[NET] Dialed discovered peer ${peerID.slice(0, 16)}`); } catch (err: any) { @@ -2190,6 +2203,13 @@ export class Network { trace(`[NET] disconnectPeer: invalid peerID ${peerID.slice(0, 16)}: ${err?.message ?? err}`); return; } + // Suppression is claimed BEFORE the first await, not after the hangUp. The two + // awaits below yield, and a `peer:discovery` event landing in that window used to + // read "not suppressed", start a dial, and have it complete after the hangUp had + // already searched for connections and found none — leaving the peer connected + // with the leave apparently finished. Recording the intent up front makes the + // window harmless: the dial that lands late sees the suppression and closes itself. + this.addRedialSuppression(networkID, peerID); // Remove the keep-alive tags FIRST so the imminent hangUp does not race // the ReconnectQueue back into a re-dial. Both tags matter: the custom // 'keep-alive-fleet' tag (peer-announce intake) and the native KEEP_ALIVE @@ -2211,10 +2231,6 @@ export class Network { } catch (err: any) { trace(`[NET] disconnectPeer: hangUp failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } - // Keep redial maintenance from re-dialing this just-left peer on the next - // status tick. Keyed by the left lishnet so rejoin lifts exactly its peers; - // cleared automatically once it reconnects legitimately. - this.addRedialSuppression(networkID, peerID); // Forget the persisted peerStore entry so the disconnect survives a restart — // suppression is in-memory only, but the peerStore is on disk. await this.purgeStalePeer(peerID, 'left-network exclusive peer'); diff --git a/backend/tests/unit/protocol/peer-discovery-handler.test.ts b/backend/tests/unit/protocol/peer-discovery-handler.test.ts index fe30046a9..1cdc7d03b 100644 --- a/backend/tests/unit/protocol/peer-discovery-handler.test.ts +++ b/backend/tests/unit/protocol/peer-discovery-handler.test.ts @@ -118,3 +118,119 @@ describe('peer:discovery — keep-alive tagging', () => { expect(dialled).toEqual([]); }); }); + +/** + * leave-network and discovery racing each other. `disconnectPeer` yields twice before it + * finishes, and a discovery event landing in that window used to read "not suppressed", + * start a dial, and complete it after the hangUp had already found nothing to close — + * leaving the peer connected with the leave apparently done. + */ +describe('peer:discovery — a dial that lands after leave-network', () => { + function racingNetwork(resumeDialInsideLeave = false) { + const hungUp: string[] = []; + const tagged: string[] = []; + const handlers = new Map(); + let releaseDial: () => void = () => {}; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).listeners = []; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).dcutrPeers = new Set(); + (network as any).peerDisconnectHandlers = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).recentDisconnects = []; + (network as any).lastMeshChange = new Map(); + (network as any).pubsub = { getTopics: () => [] }; + (network as any).peerAnnounce = { getRecentMembers: () => [] }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + addEventListener(event: string, handler: Handler) { + handlers.set(event, handler); + }, + getConnections: () => [], + getPeers: () => [], + async dial(): Promise { + await new Promise(resolve => { + releaseDial = resolve; + }); + return {}; + }, + async hangUp(pid: { toString(): string }): Promise { + hungUp.push(pid.toString()); + }, + peerStore: { + async merge(_pid: unknown, patch: { tags?: Record }): Promise { + // Only STAMPS count — leave-network merges the same keys with `undefined` + // to remove them, and that is the opposite of what these tests watch for. + const removing = Object.values(patch.tags ?? {}).some(value => value === undefined); + for (const [key, value] of Object.entries(patch.tags ?? {})) if (value !== undefined) tagged.push(key); + // This merge is disconnectPeer's first await. Letting the parked discovery + // dial finish HERE is the actual race: the handler resumes before hangUp + // has run, which is the window the ordering fix has to cover. + if (removing && resumeDialInsideLeave) { + releaseDial(); + await Bun.sleep(0); + } + }, + async delete(): Promise {}, + }, + }; + (network as any).setupEventListeners(); + const fire = (): Promise => handlers.get('peer:discovery')!({ detail: { id: { toString: () => PEER_ID }, multiaddrs: [ADDR] } }) as Promise; + return { network, hungUp, tagged, fire, releaseDial: (): void => releaseDial() }; + } + + it('claims the suppression before disconnectPeer yields', async () => { + const { network } = racingNetwork(); + const leaving = (network as any).disconnectPeer(PEER_ID, 'net-a'); + expect((network as any).redialSuppressedByNet.get('net-a')?.has(PEER_ID)).toBe(true); + await leaving; + }); + + it('closes a discovery dial that completed after the peer was left', async () => { + const { network, hungUp, tagged, fire, releaseDial } = racingNetwork(); + const discovering = fire(); // parks inside dial() + await Bun.sleep(1); + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + hungUp.length = 0; // the leave's own hangUp found nothing; watch what happens next + releaseDial(); + await discovering; + + expect(hungUp).toEqual([PEER_ID]); + expect(tagged).toEqual([]); // and never re-armed the re-dial instruction + }); + + it('keeps a late dial whose peer another joined network still needs', async () => { + const { network, hungUp, fire, releaseDial } = racingNetwork(); + const discovering = fire(); + await Bun.sleep(1); + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + (network as any).configuredBootstrapPeerIDs.add(PEER_ID); // infrastructure elsewhere + hungUp.length = 0; + releaseDial(); + await discovering; + + expect(hungUp).toEqual([]); + }); + + /** + * The interleaving itself: the discovery dial completes DURING `disconnectPeer`, after + * the tag removal and before the hangUp. Nothing later in the leave will close that + * connection, so the handler has to see the suppression already recorded. + */ + it('closes a dial that completes inside the leave, before its hangUp runs', async () => { + const { network, hungUp, tagged, fire } = racingNetwork(true); + const discovering = fire(); + await Bun.sleep(1); + + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + await discovering; + + expect(tagged).toEqual([]); + expect(hungUp).toContain(PEER_ID); + }); +}); From a9b01981c8b26e7662a0cdeaa40fdd0089e863e1 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:06:02 +0200 Subject: [PATCH 77/99] fix(network): admit a discovered identity only after it answers --- backend/src/protocol/network.ts | 16 ++++-- .../protocol/peer-eviction-guards.test.ts | 54 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index cf6646c58..fc371aa56 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1670,9 +1670,15 @@ export class Network { continue; } this.inFlightBootstrapDials.add(canonicalAddress); - // The identity set is the dedup that stops every gossip mention of the same - // peer from costing another dial, so it is claimed up front either way. - if (peerID) this.bootstrapPeerIDs.add(peerID); + // A CONFIGURED identity is user data and enters the set on the strength of the + // saved config alone. A DISCOVERED one waits for the dial: it arrived in a + // gossip message and nothing has yet shown that the identity exists, let + // alone that it is the one behind this address. Admitting it here put every + // peer ID any topic subscriber cared to name into an unbounded global set — + // one that nothing prunes, and that other code reads as "this peer is + // handled". Deduplication of repeated mentions is not this set's job and + // never was: {@link inFlightBootstrapDials} and the backoff above do that. + if (peerID && origin === 'configured') this.bootstrapPeerIDs.add(peerID); console.debug('Adding bootstrap peer:', peer); this.bootstrapTracker.markPending(networkID, peer, peerID, effectiveOrigin); try { @@ -1718,6 +1724,10 @@ export class Network { return; } if (superseded()) return; + // The peer answered, so the identity behind this address is real and + // wanted — the point at which a discovered ID has earned its place in + // the set (see the claim above for why it may not have it yet). + if (peerID) this.bootstrapPeerIDs.add(peerID); if (pidObj) { await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index f333ecda8..eb2665ba5 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1706,3 +1706,57 @@ describe('addBootstrapPeers — one dial per address at a time', () => { expect(dialled).toEqual([ADDR, ADDR]); }); }); + +/** + * `bootstrapPeerIDs` is a global, unbounded, never-TTL'd set that other code reads as + * "this peer is handled" — and the libp2p config closure reads it too. Admitting an + * identity the moment gossip named it meant any topic subscriber could put arbitrary peer + * IDs into it, before anything had shown the identity even exists. + */ +describe('addBootstrapPeers — an identity joins the bootstrap set only once it answers', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(dialFails: boolean) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (dialFails) throw new Error('dial timed out'); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + it('leaves an announced identity out while its dial is failing', async () => { + const network = bareNetwork(true); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); + + it('admits the identity once the peer answers', async () => { + const network = bareNetwork(false); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + /** A configured identity is the user's own assertion and does not wait for a dial. */ + it('admits a configured identity even when its address is down', async () => { + const network = bareNetwork(true); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); +}); From a4ab0355bea9c9b55993e5fee2cb91cfe61d95e0 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:07:56 +0200 Subject: [PATCH 78/99] fix(lishnets): drop the left network's bootstrap addresses --- backend/src/lishnet/lishnets.ts | 9 +++++ .../tests/unit/lishnet/leave-network.test.ts | 36 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 5c015f0ca..de5755927 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -249,6 +249,15 @@ export class Networks { // if it still subscribes another joined lishnet, or if it is an active circuit // relay we depend on. disconnectPeer is a safe no-op hangUp for an unconnected // peer and always strips keep-alive + suppresses redial. + // Address-level cleanup first, because the identity-level loop below cannot do it. + // One peer can be configured in two networks under two DIFFERENT addresses; on + // leaving the first, `stillConfigured` says the identity is in use elsewhere and + // skips its cleanup entirely — so the left network's own address went on counting + // as a configured bootstrap: force-dialed by the parked probe, exempt from the + // stale sweep, and disagreeing with what the UI shows as configured. + const configuredElsewhere = this.configuredBootstrapAddressesElsewhere(id); + this.network.pruneBootstrapAddresses(Networks.cleanBootstrapList(this.get(id)?.bootstrapPeers ?? []).filter(address => !configuredElsewhere.has(normalizeMultiaddrForCompare(address)))); + const stillConfigured = this.configuredBootstrapPeerIDsElsewhere(id); for (const pid of this.configuredBootstrapPeerIDsOf(id)) { if (stillConfigured.has(pid)) continue; diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index c74dbd5f6..a2fce599c 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -253,6 +253,42 @@ describe('Networks.leaveNetwork — exclusive peer disconnect', () => { expect(net.prunedBootstrap).toEqual(['pBootA']); // exemption still pruned }); + /** + * Cleanup keyed on the identity alone cannot express "this peer is configured in both + * networks, under two different addresses". The identity is in use elsewhere, so the + * whole cleanup is skipped and the left network's own address goes on counting as a + * configured bootstrap — dialed by the parked probe and exempt from removal. + */ + it('drops the left network address of a peer configured elsewhere under another address', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a', 'net-b'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + 'net-b': ['/ip4/192.0.2.2/tcp/9090/p2p/pShared'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([['/ip4/192.0.2.1/tcp/9090/p2p/pShared']]); + expect(net.prunedBootstrap).toEqual([]); // the identity itself is still configured + }); + + it('keeps an address the still-joined network configures identically', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a', 'net-b'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + 'net-b': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([[]]); + }); + + it('drops every address of a network left with nothing else configured', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA', '/ip4/192.0.2.2/tcp/9090/p2p/pAlsoA'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA', '/ip4/192.0.2.2/tcp/9090/p2p/pAlsoA']]); + }); + it('keeps a left-lishnet bootstrap peer that is still an active circuit relay', async () => { net.topicPeers.set('net-a', []); net.bootstrapOrRelay.add('pRelayNode'); // still relaying another connection From 8ca778b23e30633173e8125547f640342a00f30a Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:10:31 +0200 Subject: [PATCH 79/99] fix(network): release the probe backoff with its address --- backend/src/protocol/network.ts | 12 +++- .../protocol/peer-eviction-guards.test.ts | 58 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index fc371aa56..199f4b589 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -2023,7 +2023,14 @@ export class Network { if (addresses.length === 0) return; const drop = new Set(addresses.map(a => normalizeMultiaddrForCompare(a))); this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => !drop.has(normalizeMultiaddrForCompare(ma.toString()))); - for (const address of drop) this.configuredBootstrapAddresses.delete(address); + for (const address of drop) { + this.configuredBootstrapAddresses.delete(address); + // The pacing record goes with the address it paces. Left behind, it accumulates + // across every configuration change until stop(), and — worse — a re-added + // address inherits the old failCount and its multi-minute nextAttempt, so a user + // who deletes an entry and puts it back may see nothing dialed for minutes. + this.addressProbeBackoff.delete(address); + } } pruneConfiguredBootstrapPeer(peerID: string): void { @@ -2044,6 +2051,9 @@ export class Network { const canonical = normalizeMultiaddrForCompare(ma.toString()); if (!this.configuredBootstrapAddresses.has(canonical)) return true; this.configuredBootstrapAddresses.delete(canonical); + // Same reason as in pruneBootstrapAddresses: the pacing record belongs to the + // address, so a re-add must not inherit the deleted entry's backoff. + this.addressProbeBackoff.delete(canonical); return false; }); } diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index eb2665ba5..a6faefddc 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -383,6 +383,7 @@ describe('configured exemption ends when the peer leaves the config', () => { (network as any).bootstrapPeerIDs = new Set([PEER_ID]); (network as any).bootstrapMultiaddrs = [multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]; (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]); + (network as any).addressProbeBackoff = new Map(); (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; @@ -1760,3 +1761,60 @@ describe('addBootstrapPeers — an identity joins the bootstrap set only once it expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); }); }); + +/** + * The configured-address probe backoff is keyed by address, so it has to be released + * with the address. Left behind it grows across every configuration change, and a + * re-added address inherits the deleted entry's failCount and its multi-minute + * nextAttempt — the user deletes an entry, adds it back, and nothing dials it. + */ +describe('configured bootstrap removal releases the address probe backoff', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const OTHER = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const network = Object.create(Network.prototype) as Network; + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(ADDR), normalizeMultiaddrForCompare(OTHER)]); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(ADDR), multiaddr(OTHER)]; + (network as any).addressProbeBackoff = new Map([ + [normalizeMultiaddrForCompare(ADDR), { nextAttempt: Date.now() + 300_000, failCount: 5 }], + [normalizeMultiaddrForCompare(OTHER), { nextAttempt: Date.now() + 300_000, failCount: 5 }], + ]); + return network; + } + + it('forgets the backoff of an address removed from the configuration', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(ADDR))).toBe(false); + }); + + it('keeps the backoff of an address that stayed', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(OTHER))).toBe(true); + }); + + it('forgets the backoff of every configured address of a removed peer', () => { + const network = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect([...(network as any).addressProbeBackoff.keys()]).toEqual([]); + }); + + /** A gossip-learned address of the same peer is not the user's to lose — nor its pacing. */ + it('leaves a discovered address of the removed peer alone', () => { + const network = bareNetwork(); + (network as any).configuredBootstrapAddresses.delete(normalizeMultiaddrForCompare(OTHER)); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(OTHER))).toBe(true); + }); + + /** The user deletes an entry and puts it straight back: it must be dialed at once. */ + it('lets a re-added address be probed immediately', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).isAddressProbeDue(normalizeMultiaddrForCompare(ADDR), Date.now())).toBe(true); + }); +}); From 776663640047ce487e618decc9718c0d2bcd2cdc Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:11:50 +0200 Subject: [PATCH 80/99] fix(peer-announce): clear members and rate limiter on stop --- backend/src/protocol/peer-announce.ts | 19 ++++++++++- .../tests/unit/protocol/peer-announce.test.ts | 34 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index 773674eb3..dda5e5a0c 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -133,6 +133,11 @@ export class AnnounceRateLimiter { } return granted; } + + /** Forget every source's budget. Used when the owning manager is stopped. */ + clear(): void { + this.buckets.clear(); + } } /** Dependencies for PeerAnnounceManager. */ @@ -239,13 +244,25 @@ export class PeerAnnounceManager { }); } - /** Stop the emitter. Idempotent. Any in-flight tick will not reschedule. */ + /** + * Stop the emitter and drop everything the previous run learned. Idempotent; any + * in-flight tick will not reschedule. + * + * Both maps are per-run state. Membership is read by leave-network to decide who to + * hang up, and it is recorded against the libp2p node that was running when the peer + * was seen — carrying it into the next start() means a leave acting on the previous + * node's mesh. The rate-limiter buckets are the same kind of thing from the other + * side: an exhausted budget surviving a restart throttles a source that has not sent + * anything to THIS run yet. + */ stop(): void { this.stopped = true; if (this.timer) { clearTimeout(this.timer); this.timer = null; } + this.topicMembers.clear(); + this.rateLimiter.clear(); } /** Handle an inbound peer-announce pubsub message. */ diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 991ca39f8..ec77494be 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -399,3 +399,37 @@ describe('AnnounceRateLimiter', () => { expect(limiter.take('c', 1, 0)).toBe(0); // recently used → still exhausted }); }); + +/** + * Both maps belong to the run that filled them. Membership is recorded against the libp2p + * node that was up at the time and is what leave-network reads to decide who to hang up; + * the rate-limiter buckets are per-source budgets. Carrying either into the next start() + * makes the new run act on the old one's state. + */ +describe('PeerAnnounceManager.stop clears per-run state', () => { + const TOPIC = `${LISH_TOPIC_PREFIX}netAAAA`; + + it('forgets topic membership', () => { + const { mgr } = intakeManager(); + mgr.noteMember(TOPIC, PA_ID); + expect(mgr.getRecentMembers(TOPIC)).toEqual([PA_ID]); + + mgr.stop(); + + expect(mgr.getRecentMembers(TOPIC)).toEqual([]); + }); + + it('gives a throttled source its full budget back after a restart', async () => { + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + expect(forwarded).toHaveLength(3); // the fourth was over budget + + mgr.stop(); + mgr.start(); + await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + + expect(forwarded).toHaveLength(4); + mgr.stop(); + }); +}); From a8ad90648594dffe8c31f345d023611003c986ec Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:13:21 +0200 Subject: [PATCH 81/99] fix(peer-announce): bound raw announce input before parsing --- backend/src/protocol/peer-announce.ts | 22 ++++++++++- .../tests/unit/protocol/peer-announce.test.ts | 37 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index dda5e5a0c..504e08843 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -47,6 +47,22 @@ const PEER_ANNOUNCE_MAX_ADDRS = 32; * latency cost (sub-2 announce cycles to fill peerStore). */ const PEER_ANNOUNCE_MAX_TOTAL_ADDRS = 128; +/** + * Hard bound on RAW entries examined per announce, before anything is deduplicated. + * + * {@link PEER_ANNOUNCE_MAX_TOTAL_ADDRS} caps the UNIQUE addresses admitted, which bounds + * what intake costs downstream but not what the walk itself costs: every raw entry is + * parsed, canonicalised and routability-tested first, and a message can carry thousands + * of duplicates or unparseable values that never reach the unique cap at all. Eight times + * the unique cap leaves ample room for a legitimate announce (which is already trimmed to + * the unique cap by its emitter) while putting a ceiling on the work one message can buy. + */ +const PEER_ANNOUNCE_MAX_RAW_ADDRS = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 8; +/** + * Longest announced address we will even attempt to parse. Real multiaddrs are well under + * this; anything longer is a parser workload, not an address. + */ +const PEER_ANNOUNCE_MAX_ADDR_LENGTH = 512; /** Max addrs we take from a single known peer when including transitive list. */ const PEER_ANNOUNCE_MAX_ADDRS_PER_PEER = 3; /** @@ -288,8 +304,12 @@ export class PeerAnnounceManager { let droppedNonRoutable = 0; let droppedDuplicate = 0; let droppedAnonymous = 0; + let examined = 0; for (const a of data.multiaddrs) { - if (typeof a !== 'string' || a.length === 0) continue; + // Counted over RAW entries, so duplicates and junk are spent from the same + // budget as anything else — the unique cap alone bounds only what survives. + if (++examined > PEER_ANNOUNCE_MAX_RAW_ADDRS) break; + if (typeof a !== 'string' || a.length === 0 || a.length > PEER_ANNOUNCE_MAX_ADDR_LENGTH) continue; if (unique.size >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; try { const ma = Multiaddr(a); diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index ec77494be..6c777e536 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -433,3 +433,40 @@ describe('PeerAnnounceManager.stop clears per-run state', () => { mgr.stop(); }); }); + +/** + * The unique cap bounds what intake ADMITS, not what the walk costs. Every raw entry is + * parsed, canonicalised and routability-tested before dedup can discard it, so a message + * of thousands of duplicates or junk values bought that work unbounded. + */ +describe('PeerAnnounceManager.handle raw input bound', () => { + it('stops examining a flood of duplicates long before the end of the list', async () => { + // The unique address sits past the raw budget, so reaching it would mean the walk + // went all the way through the padding. + const { mgr, forwarded } = intakeManager(); + const dup = withID('/ip4/198.51.100.7/tcp/9090'); + const beyond = withID('/ip4/203.0.113.200/tcp/9099'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [...Array(5000).fill(dup), beyond] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[dup]]); + }); + + it('still admits a legitimate full-size announce', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: distinctAddrs(128) }, 'netAAAA', SRC_ID); + + expect(forwarded[0]!.length).toBe(128); + }); + + it('drops an over-long value without parsing it', async () => { + const { mgr, forwarded } = intakeManager(); + const bloated = `/dns4/${'a'.repeat(600)}.example.com/tcp/9090/p2p/${ANNOUNCED_ID}`; + const named = withID('/ip4/203.0.113.5/tcp/9090'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [bloated, named] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[named]]); + }); +}); From 420a122ff8add2bf849ef21473ec22421329bc0c Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:18:35 +0200 Subject: [PATCH 82/99] perf(network): group bootstrap status updates during intake --- backend/src/protocol/bootstrap-status.ts | 45 ++++ backend/src/protocol/network.ts | 11 + .../unit/protocol/bootstrap-status.test.ts | 64 ++++++ .../unit/protocol/network-disconnect.test.ts | 15 +- .../protocol/peer-eviction-guards.test.ts | 194 ++++++++++++++++-- 5 files changed, 312 insertions(+), 17 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 889a51e52..56154349f 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -9,6 +9,18 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialS */ const MAX_DISCOVERED_PER_NETWORK = 256; +/** + * While a {@link BootstrapStatusTracker.batchDebounced} frame is open, how often the + * changes accumulated so far are published. + * + * A plain batch frame publishes once, at close — which is wrong for a body that awaits a + * dial per address: the frame would stay open for the whole list, one 10 s timeout at a + * time, and the UI would show nothing until the last address settled. Flushing on this + * interval keeps the run visible while still collapsing the burst of intake mutations + * (two per address, each rebuilding the whole snapshot) into a handful of emissions. + */ +const BATCH_FLUSH_INTERVAL_MS = 75; + /** * A stored row: the status the UI sees, plus the clock {@link BootstrapStatusTracker.sweepStale} * measures against. @@ -94,6 +106,39 @@ export class BootstrapStatusTracker { return result; } + /** + * Like {@link batch}, but for an async body long enough that holding every change to + * the end would leave the UI stale: what has accumulated is published every + * {@link BATCH_FLUSH_INTERVAL_MS} while the frame is open, and once more at close. + * + * This is the shape bootstrap intake needs — it awaits a dial between each address's + * pending mark and its outcome, so the alternatives are one emission per mutation + * (what it used to do) or one emission for the entire list (a frozen UI). + */ + async batchDebounced(networkID: string, fn: () => Promise): Promise { + let frame = this.batches.get(networkID); + if (!frame) { + frame = { depth: 0, dirty: false }; + this.batches.set(networkID, frame); + } + frame.depth++; + const open = frame; + const timer = setInterval(() => { + // A frame replaced by clear() belongs to a torn-down run; publishing its + // leftovers under the same networkID would speak for whatever came next. + if (!open.dirty || this.batches.get(networkID) !== open) return; + open.dirty = false; + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + }, BATCH_FLUSH_INTERVAL_MS); + timer.unref?.(); + try { + return await fn(); + } finally { + clearInterval(timer); + this.closeBatch(networkID, open); + } + } + /** Leave one {@link batch} frame, emitting the grouped snapshot when the last one exits. */ private closeBatch(networkID: string, frame: { depth: number; dirty: boolean }): void { frame.depth--; diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 199f4b589..24d36b9b4 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1545,6 +1545,17 @@ export class Network { * additions with no owning network, in which case stats are skipped. */ async addBootstrapPeers(peers: string[], networkID: string | null = null, origin: BootstrapPeerOrigin = 'discovered'): Promise { + // Group the run's status writes. Intake performs two per address — a pending mark + // and an outcome — and each rebuilds and publishes the network's whole peer list, + // so one 128-address announce used to cost 256 snapshots and 256 WebSocket pushes + // of which the UI kept the last. The frame flushes periodically rather than only + // at close, so a list of slow dials still reports progress as it goes. + if (networkID === null) return this.dialBootstrapEntries(peers, networkID, origin); + await this.bootstrapTracker.batchDebounced(networkID, () => this.dialBootstrapEntries(peers, networkID, origin)); + } + + /** The dial loop behind {@link addBootstrapPeers}; see there for the batching wrapper. */ + private async dialBootstrapEntries(peers: string[], networkID: string | null, origin: BootstrapPeerOrigin): Promise { if (!this.node) { console.error('Network not started - cannot add bootstrap peers'); return; diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index a7c3066b3..848960116 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -466,3 +466,67 @@ describe('BootstrapStatusTracker.pruneEntries', () => { expect(seen).toEqual([[CONF_KEPT, DISCOVERED]]); }); }); + +/** + * Bootstrap intake awaits a dial between each address's pending mark and its outcome, so + * neither of the simple answers works: emitting per mutation costs a whole snapshot per + * row per address, and holding everything to the end of the list leaves the UI blank for + * as long as the dials take. + */ +describe('BootstrapStatusTracker.batchDebounced', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWBatchBatchBatchBatchBatchBatchBatchBatchBB'; + const addr = (i: number): string => `/ip4/192.0.2.${i}/tcp/9090/p2p/${PID}`; + + function tracked() { + const tracker = new BootstrapStatusTracker(); + const seen: number[] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.length)); + return { tracker, seen }; + } + + it('collapses a fast run of mutations into a single emission', async () => { + const { tracker, seen } = tracked(); + + await tracker.batchDebounced(NET, async () => { + for (let i = 0; i < 20; i++) { + tracker.markPending(NET, addr(i), PID, 'discovered'); + tracker.recordOutcome(NET, addr(i), PID, 'connected', null, null, 'discovered'); + } + }); + + expect(seen).toEqual([20]); + }); + + it('publishes progress while a slow run is still going', async () => { + const { tracker, seen } = tracked(); + + await tracker.batchDebounced(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + await Bun.sleep(200); // a dial's worth of waiting + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + + expect(seen.length).toBeGreaterThan(1); // not held back to the end + expect(seen.at(-1)).toBe(2); + }); + + it('emits nothing for a run that changed nothing', async () => { + const { tracker, seen } = tracked(); + await tracker.batchDebounced(NET, async () => {}); + expect(seen).toEqual([]); + }); + + it('propagates the body result and still closes on a throw', async () => { + const { tracker, seen } = tracked(); + + await expect( + tracker.batchDebounced(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial exploded'); + }) + ).rejects.toThrow('dial exploded'); + + expect(seen).toEqual([1]); // what it managed to change was still published + }); +}); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index 25df7c5d8..a3625173f 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -191,7 +191,12 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapMultiaddrs = bootstrapMaStrs.map(s => multiaddr(s)); (network as any).recentDisconnects = []; - (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; (network as any).node = { // Recovery reads connectivity itself rather than trusting the tick's snapshot. getPeers: () => [], @@ -268,7 +273,13 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index a6faefddc..15be1724a 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'bun:test'; import { multiaddr } from '@multiformats/multiaddr'; import { Network, isRecoveryDialDue, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; +import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; /** * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered @@ -97,7 +98,12 @@ describe('runRedialMaintenance — eviction with no reachable address', () => { (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); (network as any).configuredBootstrapAddresses = new Set(); - (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; // Only the peer itself is ever asked about; "online" means we hold a connection @@ -171,6 +177,9 @@ describe('addBootstrapPeers — only a verified address enters the peerStore', ( (network as any).inFlightBootstrapDials = new Set(); const outcomes: string[] = []; (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, markPending() {}, recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { outcomes.push(status); @@ -334,7 +343,13 @@ describe('addBootstrapPeers — forced probe only for configured addresses', () (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [{}], // already connected to this peer some other way @@ -384,7 +399,12 @@ describe('configured exemption ends when the peer leaves the config', () => { (network as any).bootstrapMultiaddrs = [multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]; (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]); (network as any).addressProbeBackoff = new Map(); - (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [] }; (network as any).hasConnectionOtherThan = () => true; @@ -470,7 +490,13 @@ describe('addBootstrapPeers — superseded bootstrap configuration', () => { (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -534,7 +560,13 @@ describe('addBootstrapPeers — a dial that lands after leave-network', () => { (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).disconnectPeer = async (peerID: string): Promise => { disconnected.push(peerID); }; @@ -629,7 +661,13 @@ describe('addBootstrapPeers — only a working discovered address joins the auto (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -719,6 +757,9 @@ describe('addBootstrapPeers — a non-routable configured entry is still configu (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, markPending() {}, recordOutcome(_n: unknown, _a: unknown, _p: unknown, status: string, message: string | null) { outcomes.push({ status, message }); @@ -792,7 +833,13 @@ describe('addBootstrapPeers — quarantine after the probe it allowed', () => { (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -879,7 +926,14 @@ describe('addBootstrapPeers — identity mismatch trims the address, not the pee (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; (network as any).purgeStalePeer = async (id: string): Promise => { purged.push(id); }; @@ -1053,7 +1107,12 @@ describe('configured origin is a property of the address, not the peer', () => { (network as any).bootstrapPeerIDs = new Set([PEER_ID]); (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED), multiaddr(DISCOVERED)]; (network as any).recentDisconnects = []; - (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; (network as any).node = { getPeers: () => [], getConnections: () => [], @@ -1113,7 +1172,12 @@ describe('Network.stop — per-run state really is per run', () => { (network as any).bootstrapMultiaddrs = []; (network as any).delayedPeerCountTimers = new Set(); (network as any).peerAnnounce = { stop() {} }; - (network as any).bootstrapTracker = { clear() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + clear() {}, + }; (network as any).node = null; (network as any).datastore = null; return network; @@ -1184,6 +1248,9 @@ describe('runZeroConnectionRecovery — connectivity is read, not remembered', ( (network as any).bootstrapMultiaddrs = (opts.addresses ?? [ADDR_A]).map(a => multiaddr(a)); (network as any).recentDisconnects = []; (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, entries: () => { churnDumps++; return []; @@ -1247,7 +1314,12 @@ describe('runZeroConnectionRecovery — a failed dial paces the next one', () => (network as any).configuredBootstrapAddresses = new Set(opts.configured ? [normalizeMultiaddrForCompare(opts.address)] : []); (network as any).bootstrapMultiaddrs = [multiaddr(opts.address)]; (network as any).recentDisconnects = []; - (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; (network as any).node = { getPeers: (): unknown[] => [], async dial(ma: { toString(): string }): Promise { @@ -1344,7 +1416,12 @@ describe('runRedialMaintenance — quarantined peers are not candidates', () => (network as any).noReachableSince = new Map(); (network as any).configuredBootstrapPeerIDs = new Set(configured ? [PEER_ID] : []); (network as any).configuredBootstrapAddresses = new Set(); - (network as any).bootstrapTracker = { deleteDiscoveredByPeerID() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; (network as any).node = { getConnections: () => [], @@ -1412,6 +1489,9 @@ describe('addBootstrapPeers — a gossip announce of a configured address', () = (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, markPending() {}, recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { outcomes.push(status); @@ -1570,7 +1650,14 @@ describe('addBootstrapPeers — discovered dials are paced by the per-peer backo (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -1647,7 +1734,14 @@ describe('addBootstrapPeers — one dial per address at a time', () => { (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -1729,7 +1823,14 @@ describe('addBootstrapPeers — an identity joins the bootstrap set only once it (network as any).bootstrapMultiaddrs = []; (network as any).bootstrapGeneration = new Map(); (network as any).inFlightBootstrapDials = new Set(); - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {}, deletePeer() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], @@ -1818,3 +1919,66 @@ describe('configured bootstrap removal releases the address probe backoff', () = expect((network as any).isAddressProbeDue(normalizeMultiaddrForCompare(ADDR), Date.now())).toBe(true); }); }); + +/** + * Intake writes two status rows per address — a pending mark and an outcome — and each + * used to rebuild and publish the network's whole peer list. A 128-address announce cost + * 256 snapshots and 256 pushes, all but the last thrown away by the UI. + */ +describe('addBootstrapPeers — status updates are grouped per run', () => { + function bareNetwork(emissions: number[]) { + const network = Object.create(Network.prototype) as Network; + const tracker = new BootstrapStatusTracker(); + tracker.setOnChange((_networkID, status) => emissions.push(status.peers.length)); + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = tracker; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const addrs = (count: number): string[] => Array.from({ length: count }, (_v, i) => `/ip4/203.0.113.${i + 1}/tcp/9090/p2p/${PEER_ID}`); + + it('publishes one snapshot for a whole announce instead of two per address', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(20), 'net-a', 'discovered'); + + expect(emissions).toEqual([20]); + }); + + it('still records every address', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(20), 'net-a', 'discovered'); + + expect((network as any).bootstrapTracker.getStatus('net-a').peers).toHaveLength(20); + }); + + /** No owning network means no status rows at all — the wrapper must not assume one. */ + it('runs unbatched when there is no network to group under', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(3), null, 'discovered'); + + expect(emissions).toEqual([]); + }); +}); From fb4707ee85847b9c254470a1e441c1509e3e6466 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:18:57 +0200 Subject: [PATCH 83/99] test(bootstrap-status): drop Array.at for the tsconfig lib target --- backend/tests/unit/protocol/bootstrap-status.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 848960116..e2c281842 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -508,7 +508,7 @@ describe('BootstrapStatusTracker.batchDebounced', () => { }); expect(seen.length).toBeGreaterThan(1); // not held back to the end - expect(seen.at(-1)).toBe(2); + expect(seen[seen.length - 1]).toBe(2); }); it('emits nothing for a run that changed nothing', async () => { From 4809271e66eef65bba9f6ad5ed170a8700065ef3 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:21:43 +0200 Subject: [PATCH 84/99] fix(network): repair the status row when a parked probe connects --- backend/src/protocol/bootstrap-status.ts | 26 +++++++++ backend/src/protocol/network.ts | 5 ++ .../unit/protocol/bootstrap-status.test.ts | 58 +++++++++++++++++++ .../protocol/peer-eviction-guards.test.ts | 25 +++++++- 4 files changed, 113 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 56154349f..184c83ec4 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -1,4 +1,5 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialStatus, type BootstrapPeerOrigin } from '@shared'; +import { canonicalMultiaddr } from './multiaddr-utils.ts'; /** * Hard ceiling on DISCOVERED (gossip-learned) rows kept per network. A hostile @@ -218,6 +219,31 @@ export class BootstrapStatusTracker { this.notify(networkID); } + /** + * Mark every row for one endpoint reachable, in every network that has one. + * + * For the loops that probe an ADDRESS rather than a network's list: they know the + * endpoint answered but not who was waiting to hear it. The parked-bootstrap probe is + * the case that matters — it is the only thing that ever retries an address the + * routability filter rejected at configure time, and it used to keep its success to + * itself, so the row stayed red for an address that had been working for hours. + * + * Matching is canonical, not by string identity: the probe walks parsed multiaddrs + * while the rows are keyed by the spelling the user typed. + */ + recordAddressReachable(address: string): void { + const target = canonicalMultiaddr(address); + for (const [networkID, peers] of this.stats) { + let changed = false; + for (const [addr, peer] of peers) { + if (peer.status === 'connected' || canonicalMultiaddr(addr) !== target) continue; + peers.set(addr, { ...peer, status: 'connected', lastError: null, actualPeerID: null, updatedAt: new Date().toISOString(), staleSince: Date.now() }); + changed = true; + } + if (changed) this.notify(networkID); + } + } + /** Bound discovered rows per network (drop the oldest) — see MAX_DISCOVERED_PER_NETWORK. */ private capDiscovered(net: Map): void { let discovered = 0; diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 24d36b9b4..18db82dca 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1421,6 +1421,11 @@ export class Network { await node.dial(ma, { signal: AbortSignal.timeout(10000), force: true }); if (epoch !== this.runEpoch) return; this.addressProbeBackoff.delete(canonical); + // Tell the UI as well. This probe is the ONLY thing that retries an address + // the routability filter rejected at configure time, so without this the row + // written when the interface was down stayed red for as long as the node ran, + // however long the address had since been working. + this.bootstrapTracker.recordAddressReachable(ma.toString()); console.log(`[NET] parked configured bootstrap reachable again: ${ma.toString()}`); } catch (err: any) { if (epoch !== this.runEpoch) return; diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index e2c281842..a3324c845 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -530,3 +530,61 @@ describe('BootstrapStatusTracker.batchDebounced', () => { expect(seen).toEqual([1]); // what it managed to change was still published }); }); + +/** + * The address-level probes know an endpoint answered but not which networks were waiting + * to hear it. The parked-bootstrap probe is the case that matters: it is the only thing + * that retries an address the routability filter rejected at configure time, so the red + * row it left behind had no other way back to green. + */ +describe('BootstrapStatusTracker.recordAddressReachable', () => { + const NET_A = 'netAAAA'; + const NET_B = 'netBBBB'; + const PID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const ADDR = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PID}`; + const OTHER = `/ip4/192.0.2.50/tcp/9090/p2p/${PID}`; + + function seeded() { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, ADDR, PID, 'error', 'address is not routable from this host', null, 'configured'); + tracker.recordOutcome(NET_B, ADDR, PID, 'error', 'address is not routable from this host', null, 'configured'); + tracker.recordOutcome(NET_A, OTHER, PID, 'timeout', 'The operation timed out', null, 'configured'); + return tracker; + } + + const statusOf = (tracker: BootstrapStatusTracker, networkID: string, addr: string): string | undefined => tracker.getStatus(networkID)?.peers.find(p => p.multiaddr === addr)?.status; + + it('repairs the row in every network that configured the address', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(statusOf(tracker, NET_A, ADDR)).toBe('connected'); + expect(statusOf(tracker, NET_B, ADDR)).toBe('connected'); + }); + + it('clears the error text it is replacing', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(tracker.getStatus(NET_A)?.peers.find(p => p.multiaddr === ADDR)?.lastError).toBe(null); + }); + + it('leaves other addresses of the same peer alone', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(statusOf(tracker, NET_A, OTHER)).toBe('timeout'); + }); + + /** The probe walks parsed multiaddrs; the rows keep the spelling the user typed. */ + it('matches the row canonically, not by string identity', () => { + const tracker = seeded(); + tracker.recordAddressReachable(`/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PID}`); + expect(statusOf(tracker, NET_A, ADDR)).toBe('connected'); + }); + + it('emits nothing when no row is waiting for that address', () => { + const tracker = seeded(); + const seen: string[] = []; + tracker.setOnChange(networkID => seen.push(networkID)); + tracker.recordAddressReachable(`/ip4/198.51.100.77/tcp/9090/p2p/${PID}`); + expect(seen).toEqual([]); + }); +}); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 15be1724a..708df8394 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1000,6 +1000,12 @@ describe('probeParkedConfiguredBootstraps', () => { (network as any).redialBackoff = new Map(); (network as any).addressProbeBackoff = new Map(); (network as any).bootstrapMultiaddrs = addresses.map(a => multiaddr(a)); + const repaired: string[] = []; + (network as any).bootstrapTracker = { + recordAddressReachable(address: string) { + repaired.push(address); + }, + }; (network as any).node = { getConnections: () => (opts.connectionAddrs ?? []).map(a => ({ remoteAddr: { toString: () => a } })), async dial(ma: { toString(): string }): Promise { @@ -1007,7 +1013,7 @@ describe('probeParkedConfiguredBootstraps', () => { if (opts.failAddresses?.includes(ma.toString())) throw new Error('dial timeout'); }, }; - return { network, dialed }; + return { network, dialed, repaired }; } const run = (network: Network): Promise => (network as any).probeParkedConfiguredBootstraps(1); @@ -1018,6 +1024,23 @@ describe('probeParkedConfiguredBootstraps', () => { expect(dialed).toEqual([multiaddr(PARKED).toString()]); }); + /** + * This probe is the only thing that ever retries an address the routability filter + * rejected at configure time — a LAN or VPN bootstrap whose interface was down. The + * `error` row written back then had no other way to go green again. + */ + it('repairs the status row when a parked address answers', async () => { + const { network, repaired } = bareNetwork(); + await run(network); + expect(repaired).toEqual([multiaddr(PARKED).toString()]); + }); + + it('leaves the row alone while the address is still failing', async () => { + const { network, repaired } = bareNetwork({ failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + expect(repaired).toEqual([]); + }); + it('leaves a discovered address to the loops that own it', async () => { const { network, dialed } = bareNetwork({ configured: false }); await run(network); From 0339f1df5e593699c06226e13c6399785cc1e2a8 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:23:23 +0200 Subject: [PATCH 85/99] fix(network): cap the autodial address list --- backend/src/protocol/network.ts | 20 +++++++++ .../protocol/peer-eviction-guards.test.ts | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 18db82dca..c1980da2f 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -136,6 +136,18 @@ const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; */ const CONFIGURED_PROBE_BACKOFF_MAX_MS = 5 * 60_000; +/** + * Ceiling on the autodial list zero-connection recovery walks. + * + * A discovered address earns its place by answering a dial, so the list grows with the + * number of distinct endpoints that have ever worked — unbounded on a fleet with churn, + * and the array is otherwise only ever shortened by an identity purge. Over the ceiling + * the OLDEST DISCOVERED entry goes: configured entries are finite user data and the way + * back into a network, so they are never the ones dropped. 512 is far above any real + * node's working set while keeping the list, and the walk over it, bounded. + */ +const MAX_BOOTSTRAP_ADDRESSES = 512; + /** * Where the eviction window should run from after a re-dial failure. * @@ -2026,6 +2038,14 @@ export class Network { const canonical = normalizeMultiaddrForCompare(ma.toString()); if (this.bootstrapMultiaddrs.some(m => normalizeMultiaddrForCompare(m.toString()) === canonical)) return; this.bootstrapMultiaddrs.push(ma); + if (this.bootstrapMultiaddrs.length <= MAX_BOOTSTRAP_ADDRESSES) return; + // Array order is insertion order, so the first discovered entry is the oldest one. + // A list of nothing but configured entries is left to grow: it is bounded by what + // the user saved, and dropping any of it would silently unconfigure a bootstrap. + const oldestDiscovered = this.bootstrapMultiaddrs.findIndex(m => !this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(m.toString()))); + if (oldestDiscovered === -1) return; + trace(`[NET] autodial list full — dropping ${this.bootstrapMultiaddrs[oldestDiscovered]?.toString()}`); + this.bootstrapMultiaddrs.splice(oldestDiscovered, 1); } /** diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index 708df8394..fc23c7057 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -2005,3 +2005,47 @@ describe('addBootstrapPeers — status updates are grouped per run', () => { expect(emissions).toEqual([]); }); }); + +/** + * The autodial list grows with every discovered endpoint that has ever answered, and + * nothing but an identity purge ever shortens it — so a network with churn of reachable + * one-off peers inflates it for the lifetime of the process, and zero-connection recovery + * walks the whole thing. + */ +describe('the autodial address list is bounded', () => { + const discovered = (i: number): string => `/ip4/198.51.100.${i % 254}/tcp/${9000 + i}/p2p/${PEER_ID}`; + const CONFIGURED = `/ip4/203.0.113.1/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(configured: string[] = []) { + const network = Object.create(Network.prototype) as Network; + (network as any).bootstrapMultiaddrs = []; + (network as any).configuredBootstrapAddresses = new Set(configured.map(a => normalizeMultiaddrForCompare(a))); + for (const address of configured) (network as any).rememberBootstrapAddress(multiaddr(address)); + return network; + } + + const remember = (network: Network, count: number): void => { + for (let i = 0; i < count; i++) (network as any).rememberBootstrapAddress(multiaddr(discovered(i))); + }; + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('stops growing past the ceiling', () => { + const network = bareNetwork(); + remember(network, 600); + expect((network as any).bootstrapMultiaddrs).toHaveLength(512); + }); + + it('drops the oldest discovered entry, keeping the newest', () => { + const network = bareNetwork(); + remember(network, 600); + expect(addresses(network)).not.toContain(multiaddr(discovered(0)).toString()); + expect(addresses(network)).toContain(multiaddr(discovered(599)).toString()); + }); + + /** Configured entries are the user's way back into a network and are never evicted. */ + it('never drops a configured address to make room', () => { + const network = bareNetwork([CONFIGURED]); + remember(network, 600); + expect(addresses(network)).toContain(multiaddr(CONFIGURED).toString()); + }); +}); From 26d63e163f39ac43084d98da6e0722dc1e6fb600 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 03:26:09 +0200 Subject: [PATCH 86/99] fix(lishnets): reset bootstrap status when leaving a network --- backend/src/lishnet/lishnets.ts | 10 +++++++--- .../tests/unit/lishnet/leave-network.test.ts | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index de5755927..72fd1eb23 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -219,10 +219,14 @@ export class Networks { this.network.unsubscribeTopic(id); this.joinedNetworks.delete(id); - // Abandon any bootstrap job still walking this network's list. Left half-way + // Abandon any bootstrap job still walking this network's list — left half-way // through, it would keep dialing peers of a network we just left and clear the - // redial suppression the loop below is about to apply. - this.network.bumpBootstrapGeneration(id); + // redial suppression the loop below is about to apply — and drop the status rows + // with it. Those rows describe a membership that has ended: keeping them meant a + // later rejoin opened on the previous session's connected/error/discovered results + // until fresh dials happened to overwrite each one. Reset publishes an empty list, + // which is what the UI should show for a network this node is not in. + this.network.resetBootstrapStatus(id); // Subscribers of any OTHER joined lishnet must stay connected (shared // infrastructure). Compute this set BEFORE the bootstrap cleanup so that loop diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index a2fce599c..4087de2ce 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -28,6 +28,8 @@ interface MockNet { pruneConfiguredBootstrapPeer(pid: string): void; bumpBootstrapGeneration(networkID: string): void; generationBumps: string[]; + resetBootstrapStatus(networkID: string): void; + statusResets: string[]; pruneBootstrapAddresses(addresses: string[]): void; prunedAddresses: string[][]; pruneBootstrapStatus(networkID: string, keep: string[]): void; @@ -49,6 +51,7 @@ function makeMockNet(): MockNet { prunedBootstrap: [], suppressionClearedFor: [], generationBumps: [], + statusResets: [], prunedAddresses: [], prunedStatus: [], dialledLists: [], @@ -78,6 +81,9 @@ function makeMockNet(): MockNet { bumpBootstrapGeneration(networkID) { this.generationBumps.push(networkID); }, + resetBootstrapStatus(networkID) { + this.statusResets.push(networkID); + }, pruneBootstrapAddresses(addresses) { this.prunedAddresses.push(addresses); }, @@ -289,6 +295,18 @@ describe('Networks.leaveNetwork — exclusive peer disconnect', () => { expect(net.prunedAddresses).toEqual([['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA', '/ip4/192.0.2.2/tcp/9090/p2p/pAlsoA']]); }); + /** + * The rows describe a membership that has ended. Kept, a later rejoin opens on the + * previous session's connected/error/discovered results until fresh dials happen to + * overwrite each one. + */ + it('drops the bootstrap status of the network it left', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a'], { 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA'] }); + await leave(networks, 'net-a'); + expect(net.statusResets).toEqual(['net-a']); + }); + it('keeps a left-lishnet bootstrap peer that is still an active circuit relay', async () => { net.topicPeers.set('net-a', []); net.bootstrapOrRelay.add('pRelayNode'); // still relaying another connection From baa49286daf667167799a3929476459022fd65ca Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 06:47:00 +0200 Subject: [PATCH 87/99] fix(network): serialise start/stop with a lifecycle state --- backend/src/protocol/network.ts | 110 ++++++++++++-- .../unit/protocol/network-lifecycle.test.ts | 143 ++++++++++++++++++ .../protocol/peer-eviction-guards.test.ts | 5 + 3 files changed, 242 insertions(+), 16 deletions(-) create mode 100644 backend/tests/unit/protocol/network-lifecycle.test.ts diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index c1980da2f..bec514a43 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1,4 +1,5 @@ import { createLibp2p } from 'libp2p'; +import { Mutex } from 'async-mutex'; import { KEEP_ALIVE } from '@libp2p/interface'; import { SqliteDatastore } from './datastore.ts'; import { privateKeyToProtobuf } from '@libp2p/crypto/keys'; @@ -215,11 +216,30 @@ export function shouldEvictUnreachablePeer(input: { reachable: boolean; failCoun */ const MAX_PUBSUB_PAYLOAD_BYTES = 256 * 1024; +/** + * Where the node is in its start/stop cycle. + * + * `this.node` alone cannot express this: it is set half-way through {@link Network.start} + * and stays set for the whole of {@link Network.stop}, so a failed start left a node + * object that never started looking exactly like a running one, and a caller starting + * during a stop was told "already running" and then had its node torn down under it. + * Only a fully successful start reaches `running`. + */ +export type NetworkLifecycle = 'stopped' | 'starting' | 'running' | 'stopping'; + /** * Single shared libp2p node. * LISH networks are logical groups represented as pubsub topics on this one node. */ export class Network { + private lifecycle: NetworkLifecycle = 'stopped'; + /** + * Serialises start() against stop(). Both mutate the same fields across several + * awaits, and without this two concurrent start() calls could both pass the + * "already running" check before either created a node — two libp2p instances over + * one identity and one SQLite datastore. + */ + private readonly lifecycleMutex = new Mutex(); private node: Libp2p | null = null; private pubsub: PubSub | null = null; private datastore: SqliteDatastore | null = null; @@ -574,11 +594,33 @@ export class Network { * @param bootstrapPeers - merged list of bootstrap peers from all enabled lishnets */ async start(bootstrapPeers: string[] = []): Promise { - if (this.node) { - console.log('Network node is already running'); - return; - } + // Serialised against stop() and against another start(): every field below is + // touched across awaits by both, so overlapping runs would interleave into two + // nodes over one datastore, or into a start whose node a concurrent stop tears + // down while its caller is told the start succeeded. + await this.lifecycleMutex.runExclusive(async () => { + if (this.lifecycle !== 'stopped') { + console.log('Network node is already running'); + return; + } + this.lifecycle = 'starting'; + try { + await this.startLocked(bootstrapPeers); + this.lifecycle = 'running'; + } catch (err) { + // A half-built start owns a datastore handle and possibly a libp2p node. + // Leaving either behind is what made a failed start unrecoverable without + // restarting the process: the SQLite file stayed locked and `this.node` + // stayed set, so the next start reported "already running" forever. + await this.teardown(); + this.lifecycle = 'stopped'; + throw err; + } + }); + } + /** The body of {@link start}, run under the lifecycle mutex. */ + private async startLocked(bootstrapPeers: string[]): Promise { // Read settings const allSettings = this.settings.list(); @@ -1540,7 +1582,12 @@ export class Network { * Whether the node is running. */ isRunning(): boolean { - return this.node !== null; + return this.lifecycle === 'running'; + } + + /** Current start/stop phase. Exposed for tests and diagnostics. */ + getLifecycle(): NetworkLifecycle { + return this.lifecycle; } /** @@ -2684,6 +2731,28 @@ export class Network { } async stop(): Promise { + // Same mutex as start(): a stop that overlapped a start used to tear down a node + // the start had just handed back to its caller as successfully started. + await this.lifecycleMutex.runExclusive(async () => { + this.lifecycle = 'stopping'; + try { + await this.teardown(); + } finally { + this.lifecycle = 'stopped'; + } + }); + } + + /** + * Release everything a run owns. Shared by {@link stop} and by the failure path of + * {@link start}, because a half-built start holds the same resources a finished one + * does — and leaving them behind is what made a failed start unrecoverable. + * + * Every step is best-effort and the field nulling happens in a `finally`: a node that + * refuses to stop must not also cost us the datastore handle and leave the instance + * permanently wedged in "running". + */ + private async teardown(): Promise { this.runEpoch++; // invalidate any in-flight status tick before touching state if (this.statusInterval) { clearInterval(this.statusInterval); @@ -2741,18 +2810,27 @@ export class Network { this.delayedPeerCountTimers.clear(); this.redialSuppressedByNet.clear(); this.pxIngressLogKeys.clear(); - if (this.node) { - await this.node.stop(); - console.log('Network stopped'); - } - if (this.datastore) { - await this.datastore.close(); - console.log('Datastore closed'); + try { + if (this.node) { + await this.node.stop(); + console.log('Network stopped'); + } + } catch (err: any) { + trace(`[NET] node.stop() failed: ${err?.message ?? err}`); + } finally { + try { + if (this.datastore) { + await this.datastore.close(); + console.log('Datastore closed'); + } + } catch (err: any) { + trace(`[NET] datastore.close() failed: ${err?.message ?? err}`); + } + this.node = null; + this.pubsub = null; + this.datastore = null; + this.currentPrivateKey = null; } - this.node = null; - this.pubsub = null; - this.datastore = null; - this.currentPrivateKey = null; } async cliFindPeer(peerID: string): Promise { diff --git a/backend/tests/unit/protocol/network-lifecycle.test.ts b/backend/tests/unit/protocol/network-lifecycle.test.ts new file mode 100644 index 000000000..71da94acc --- /dev/null +++ b/backend/tests/unit/protocol/network-lifecycle.test.ts @@ -0,0 +1,143 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * Unit tests for the transactional start()/stop() lifecycle. + * + * The node used to be considered "running" the moment `this.node` was assigned — + * half-way through start(), and for the whole of stop(). That made four things + * possible, each covered below: a phantom running state after a failed start, a + * leaked datastore handle, two concurrent starts both building a node, and a stop + * that abandoned the datastore when node.stop() threw. + */ + +/** A Network with no libp2p behind it — start()/stop() orchestration only. */ +function bareNetwork(): Network { + return new Network('/nonexistent/data-dir', {} as any, { list: (): any => ({}) } as any); +} + +/** A promise plus the handles to settle it, so a test can hold a step open. */ +function deferred(): { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } { + let resolve!: (v: T) => void; + let reject!: (e: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe('Network lifecycle', () => { + it('a second start() while the first is still building does not build a second node', async () => { + const net = bareNetwork(); + const gate = deferred(); + let started = 0; + (net as any).startLocked = async (): Promise => { + started++; + await gate.promise; + }; + + const first = net.start([]); + // Second caller arrives while the first is parked inside startLocked — exactly the + // window in which `if (this.node)` was still false and let both through. + const second = net.start([]); + await Promise.resolve(); + expect(started).toBe(1); + + gate.resolve(); + await Promise.all([first, second]); + expect(started).toBe(1); + expect(net.isRunning()).toBe(true); + expect(net.getLifecycle()).toBe('running'); + }); + + it('a failed start leaves nothing running and does not block the next start', async () => { + const net = bareNetwork(); + let torndown = 0; + (net as any).teardown = async (): Promise => { + torndown++; + }; + (net as any).startLocked = async (): Promise => { + throw new Error('createLibp2p exploded'); + }; + + await expect(net.start([])).rejects.toThrow('createLibp2p exploded'); + expect(torndown).toBe(1); + expect(net.isRunning()).toBe(false); + expect(net.getLifecycle()).toBe('stopped'); + + // The whole point: a start that failed must be retryable in-process. + (net as any).startLocked = async (): Promise => {}; + await net.start([]); + expect(net.isRunning()).toBe(true); + }); + + it('start() is not reported as running until it has fully finished', async () => { + const net = bareNetwork(); + const gate = deferred(); + (net as any).startLocked = async (): Promise => { + // Whatever a partially built start has assigned, it is not running yet. + (net as any).node = { stop: async (): Promise => {} }; + await gate.promise; + }; + + const running = net.start([]); + await Promise.resolve(); + expect(net.isRunning()).toBe(false); + expect(net.getLifecycle()).toBe('starting'); + + gate.resolve(); + await running; + expect(net.isRunning()).toBe(true); + }); + + it('stop() waits for an in-flight start instead of tearing it down mid-build', async () => { + const net = bareNetwork(); + const gate = deferred(); + const order: string[] = []; + (net as any).startLocked = async (): Promise => { + order.push('start:begin'); + await gate.promise; + order.push('start:end'); + }; + (net as any).teardown = async (): Promise => { + order.push('teardown'); + }; + + const starting = net.start([]); + await Promise.resolve(); + const stopping = net.stop(); + // Give the stop every chance to cut in front of the unfinished start. + await Promise.resolve(); + await Promise.resolve(); + expect(order).toEqual(['start:begin']); + + gate.resolve(); + await Promise.all([starting, stopping]); + expect(order).toEqual(['start:begin', 'start:end', 'teardown']); + expect(net.getLifecycle()).toBe('stopped'); + }); + + it('a node that refuses to stop still costs neither the datastore nor the reset', async () => { + const net = bareNetwork(); + let closed = 0; + (net as any).node = { + stop: async (): Promise => { + throw new Error('node.stop failed'); + }, + }; + (net as any).datastore = { + close: async (): Promise => { + closed++; + }, + }; + + await net.stop(); + + expect(closed).toBe(1); + expect((net as any).node).toBeNull(); + expect((net as any).datastore).toBeNull(); + expect(net.getLifecycle()).toBe('stopped'); + expect(net.isRunning()).toBe(false); + }); +}); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts index fc23c7057..a61518f56 100644 --- a/backend/tests/unit/protocol/peer-eviction-guards.test.ts +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { multiaddr } from '@multiformats/multiaddr'; +import { Mutex } from 'async-mutex'; import { Network, isRecoveryDialDue, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; @@ -1203,6 +1204,10 @@ describe('Network.stop — per-run state really is per run', () => { }; (network as any).node = null; (network as any).datastore = null; + // stop() runs under the lifecycle mutex; a prototype-only instance has no field + // initializers, so it has to be supplied here. + (network as any).lifecycle = 'running'; + (network as any).lifecycleMutex = new Mutex(); return network; } From 381a1b4c198bba5afc6f6a3a6070222f267078b4 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 06:50:34 +0200 Subject: [PATCH 88/99] fix(network): bind disconnectPeer to the node and epoch it started on --- backend/src/protocol/network.ts | 28 ++- .../protocol/network-restart-safety.test.ts | 173 ++++++++++++++++++ 2 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 backend/tests/unit/protocol/network-restart-safety.test.ts diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index bec514a43..6b67bff53 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1793,9 +1793,18 @@ export class Network { // by leave-network and sits in the suppression set, or we left the network // this dial belonged to before ever seeing the peer as one of its members, // in which case it never entered that set at all. + // + // The EPOCH is checked first and separately from the full `superseded()`: + // a different node instance means none of this run's suppression or + // subscription state describes the node that would be torn at, so the + // destructive branch must not run at all. A bumped GENERATION is the + // opposite case — the network was left or its list replaced on the SAME + // node, which is precisely when this connection needs closing, so it may + // not short-circuit ahead of it. + if (epoch !== this.runEpoch) return; if (peerID && networkID && (this.isRedialSuppressed(peerID) || !this.isTopicSubscribed(networkID)) && !this.isPeerNeededByJoinedNetwork(peerID)) { trace(`[NET] bootstrap dial landed after leave, disconnecting: ${peerID.slice(0, 16)}`); - await this.disconnectPeer(peerID, networkID); + await this.disconnectPeer(peerID, networkID, epoch); return; } if (superseded()) return; @@ -2296,9 +2305,15 @@ export class Network { * * `networkID` is the lishnet the peer is being left with — the peer is suppressed * under it so rejoining that lishnet lifts exactly its peers. + * + * `epoch` binds the whole sequence to the node instance it started on, for the same + * reason {@link purgeStalePeer} takes one: this awaits twice, and re-reading + * `this.node` after a stop()/start() used to hand back the NEW node — so the hangUp + * and the purge landed on an instance that had never heard of this leave. */ - async disconnectPeer(peerID: string, networkID: string): Promise { - if (!this.node) return; + async disconnectPeer(peerID: string, networkID: string, epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; let pid: PeerID; try { pid = peerIDFromString(peerID); @@ -2322,21 +2337,22 @@ export class Network { // undefined as the tag value removes it (per @libp2p/interface PeerStore // merge semantics). try { - await this.node.peerStore.merge(pid, { + await node.peerStore.merge(pid, { tags: { 'keep-alive-fleet': undefined, [KEEP_ALIVE]: undefined }, }); } catch (err: any) { trace(`[NET] disconnectPeer: tag removal failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } + if (epoch !== this.runEpoch) return; try { - await this.node.hangUp(pid); + await node.hangUp(pid); trace(`[NET] disconnectPeer: hung up ${peerID.slice(0, 16)}`); } catch (err: any) { trace(`[NET] disconnectPeer: hangUp failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } // Forget the persisted peerStore entry so the disconnect survives a restart — // suppression is in-memory only, but the peerStore is on disk. - await this.purgeStalePeer(peerID, 'left-network exclusive peer'); + await this.purgeStalePeer(peerID, 'left-network exclusive peer', epoch); } /** Snapshot of all per-network bootstrap statuses. */ diff --git a/backend/tests/unit/protocol/network-restart-safety.test.ts b/backend/tests/unit/protocol/network-restart-safety.test.ts new file mode 100644 index 000000000..a87566019 --- /dev/null +++ b/backend/tests/unit/protocol/network-restart-safety.test.ts @@ -0,0 +1,173 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * A destructive operation started on one libp2p node must never finish against the + * next one. Both paths below await in the middle, and both used to re-read + * `this.node` (or re-evaluate the run epoch) afterwards — so a stop()/start() + * landing in that window handed them the NEW node to hang up, purge and evict on. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const ADDR = `/ip4/192.0.2.10/tcp/9090/p2p/${PEER_ID}`; + +function deferred(): { promise: Promise; resolve: (v: T) => void } { + let resolve!: (v: T) => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +describe('Network.disconnectPeer — bound to the node it started on', () => { + function harness() { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map>(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + + const gate = deferred(); + const touchedByOld: string[] = []; + const nodeA = { + getConnections: (): unknown[] => [], + async hangUp(): Promise { + touchedByOld.push('A.hangUp'); + }, + peerStore: { + async merge(): Promise { + await gate.promise; + }, + async delete(): Promise { + touchedByOld.push('A.delete'); + }, + }, + }; + const touchedByNew: string[] = []; + const nodeB = { + getConnections: (): unknown[] => [], + async hangUp(): Promise { + touchedByNew.push('B.hangUp'); + }, + peerStore: { + async merge(): Promise {}, + async delete(): Promise { + touchedByNew.push('B.delete'); + }, + }, + }; + (network as any).node = nodeA; + return { network, nodeB, gate, touchedByOld, touchedByNew }; + } + + it('does not hang up or purge on the node that replaced it mid-flight', async () => { + const { network, nodeB, gate, touchedByNew } = harness(); + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + // stop() + start(): a new epoch and a new node, exactly as a restart leaves them. + (network as any).runEpoch = 2; + (network as any).node = nodeB; + gate.resolve(); + await leaving; + + expect(touchedByNew).toEqual([]); + }); + + it('does not purge on the new node when the restart lands during the hangUp', async () => { + const { network, nodeB, gate, touchedByNew } = harness(); + const hangUpGate = deferred(); + (network as any).node.hangUp = async (): Promise => { + await hangUpGate.promise; + }; + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + gate.resolve(); + await Promise.resolve(); + await Promise.resolve(); + (network as any).runEpoch = 2; + (network as any).node = nodeB; + hangUpGate.resolve(); + await leaving; + + expect(touchedByNew).toEqual([]); + }); + + it('still completes normally when no restart happens', async () => { + const { network, gate, touchedByOld } = harness(); + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + gate.resolve(); + await leaving; + + expect(touchedByOld).toEqual(['A.hangUp', 'A.delete']); + }); +}); + +describe('Network.addBootstrapPeers — a dial that lands after a restart', () => { + function harness() { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + // The peer is suppressed, so a dial landing on THIS run would legitimately take + // the destructive "landed after leave" branch. After a restart it must not. + (network as any).redialSuppressedByNet = new Map([['net-a', new Set([PEER_ID])]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending(): void {}, + recordOutcome(): void {}, + }; + const gate = deferred<{ remoteAddr: string }>(); + (network as any).node = { + peerId: { toString: (): string => 'selfID' }, + getConnections: (): unknown[] => [], + dial: (): Promise<{ remoteAddr: string }> => gate.promise, + peerStore: { async merge(): Promise {} }, + }; + const disconnected: string[] = []; + (network as any).disconnectPeer = async (pid: string): Promise => { + disconnected.push(pid); + }; + return { network, gate, disconnected }; + } + + it('does not disconnect on the new node when the epoch moved on', async () => { + const { network, gate, disconnected } = harness(); + + const dialing = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + (network as any).runEpoch = 2; + gate.resolve({ remoteAddr: ADDR }); + await dialing; + + expect(disconnected).toEqual([]); + }); + + it('still disconnects when the dial lands on the same run after a leave', async () => { + const { network, gate, disconnected } = harness(); + // A leave bumps the generation; that must NOT short-circuit ahead of the + // disconnect — the connection this dial just opened is the thing to close. + (network as any).bootstrapGeneration = new Map([['net-a', 1]]); + + const dialing = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + (network as any).bootstrapGeneration.set('net-a', 2); + gate.resolve({ remoteAddr: ADDR }); + await dialing; + + expect(disconnected).toEqual([PEER_ID]); + }); +}); From 30bde0db312e846293cec753ec3286a1babbc036 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 06:59:10 +0200 Subject: [PATCH 89/99] fix(lishnets): serialise enable and disable per lishnet --- backend/src/lishnet/lishnets.ts | 107 +++++++++-- .../unit/lishnet/enable-serialisation.test.ts | 176 ++++++++++++++++++ .../tests/unit/lishnet/leave-network.test.ts | 9 +- 3 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 backend/tests/unit/lishnet/enable-serialisation.test.ts diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index 72fd1eb23..b2f1cc257 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -1,4 +1,5 @@ import { type Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; import { Network, normalizeMultiaddrForCompare } from '../protocol/network.ts'; import { canonicalMultiaddr } from '../protocol/multiaddr-utils.ts'; import { Utils } from '../utils.ts'; @@ -18,6 +19,39 @@ export class Networks { // Track which lishnets are currently joined (subscribed) private joinedNetworks: Set = new Set(); + /** + * One lock per lishnet, so join and leave of the SAME network never interleave. + * + * Both of them await for a long time — a join waits on bootstrap dials, a leave + * disconnects peers one at a time — while mutating `joinedNetworks`, the pubsub + * subscription and the redial suppression that the other one reads. Overlapping + * runs left those four disagreeing with the database and with each other. + */ + private readonly networkOperations = new Map(); + /** + * The revision of the LAST enable/disable requested for a network, bumped + * synchronously when the request arrives. + * + * The lock alone only orders the operations; it cannot tell a queued one that the + * user has since asked for the opposite. Each operation carries the revision it was + * created for and abandons itself — before it starts, after each await and before + * every callback — once a newer one exists. That is what keeps the callbacks, the + * subscription and the database describing the same, final, request. + */ + private readonly desiredRevisions = new Map(); + /** + * The join/leave state last announced to higher layers, per lishnet. + * + * A superseded operation must not announce anything, but the one that settles the + * network must — and it can find the runtime already in the state it wanted, because + * an abandoned predecessor got part of the way there. Announcing the OUTCOME rather + * than the operation covers both: no event for a change that was undone before it + * settled, exactly one for a change that stuck. Unset reads as "not joined", and the + * startup join seeds it directly — startup itself stays silent (it has its own + * resume path) while a later disable still has a `true` to change away from. + */ + private readonly announcedJoined = new Map(); + // Callback for peer count changes private _onPeerCountChange: ((counts: { networkID: string; count: number }[]) => void) | null = null; // Callback for bootstrap status changes @@ -107,6 +141,9 @@ export class Networks { for (const net of enabled) { this.network.subscribeTopic(net.networkID); this.joinedNetworks.add(net.networkID); + // Startup itself announces nothing, but a later disable has to have a joined + // state to change away from — otherwise its leave looks like a no-op. + this.announcedJoined.set(net.networkID, true); if (net.bootstrapPeers.length > 0) { // Fire-and-forget so a slow / unreachable network does not delay startup of the others. this.network.addBootstrapPeers(net.bootstrapPeers, net.networkID, 'configured').catch(err => { @@ -124,15 +161,59 @@ export class Networks { if (!lishnetExists(this.db, id)) return false; setLISHnetEnabled(this.db, id, enabled); - - if (enabled) await this.joinNetwork(id); - else await this.leaveNetwork(id); + await this.reconcile(id, enabled); return true; } + /** The lock guarding one lishnet's join/leave — see {@link networkOperations}. */ + private operationLock(id: string): Mutex { + let lock = this.networkOperations.get(id); + if (!lock) { + lock = new Mutex(); + this.networkOperations.set(id, lock); + } + return lock; + } + + /** + * Bring the runtime in line with a requested enabled state, serialised per lishnet. + * + * The revision is claimed SYNCHRONOUSLY, before anything is awaited, so the order in + * which requests arrive — not the order in which their dials happen to finish — is + * what decides the outcome. A request that has been overtaken by a newer one does + * nothing at all: three fast toggles cost one operation, the last one. + */ + private async reconcile(id: string, enabled: boolean): Promise { + const revision = (this.desiredRevisions.get(id) ?? 0) + 1; + this.desiredRevisions.set(id, revision); + await this.operationLock(id).runExclusive(async () => { + if (this.desiredRevisions.get(id) !== revision) return; + if (enabled) await this.joinNetwork(id); + else await this.leaveNetwork(id, revision); + if (this.desiredRevisions.get(id) !== revision) return; + this.announce(id, this.joinedNetworks.has(id)); + }); + } + + /** True while `revision` is still the newest request for this lishnet. */ + private isCurrentRevision(id: string, revision: number | undefined): boolean { + return revision === undefined || this.desiredRevisions.get(id) === revision; + } + + /** Tell higher layers about a settled join/leave, once per actual change. */ + private announce(id: string, joined: boolean): void { + if ((this.announcedJoined.get(id) ?? false) === joined) return; + this.announcedJoined.set(id, joined); + if (joined) this._onNetworkJoined?.(id); + else this._onNetworkLeft?.(id); + } + /** * Join a lishnet (subscribe to its topic, add bootstrap peers). + * + * Announcing the join is {@link reconcile}'s job, not this one's — see + * {@link announcedJoined} for why the outcome is what gets announced. */ private async joinNetwork(id: string): Promise { if (this.joinedNetworks.has(id)) { @@ -157,10 +238,6 @@ export class Networks { if (net && net.bootstrapPeers.length > 0) await this.network.addBootstrapPeers(net.bootstrapPeers, id, 'configured'); console.log(`✓ Joined lishnet: ${net?.name ?? id}`); - - // Notify higher layers (e.g. transfer) so downloads suspended when this - // lishnet was last left can resume now that it is joined again. - this._onNetworkJoined?.(id); } /** @@ -205,7 +282,11 @@ export class Networks { return out; } - private async leaveNetwork(id: string): Promise { + /** + * Leave a lishnet. `revision` is the disable request this leave belongs to; see + * {@link joinNetwork} for why the long loops below re-check it. + */ + private async leaveNetwork(id: string, revision?: number): Promise { if (!this.joinedNetworks.has(id)) return; // Snapshot the topic subscribers BEFORE unsubscribing — unsubscribeTopic @@ -264,6 +345,11 @@ export class Networks { const stillConfigured = this.configuredBootstrapPeerIDsElsewhere(id); for (const pid of this.configuredBootstrapPeerIDsOf(id)) { + // Each disconnect awaits a hangUp and a peerStore delete, so a long list keeps + // this loop running well past the point at which the user may have re-enabled + // the lishnet. Every peer torn down after that belongs to the network we are + // about to be back in. + if (!this.isCurrentRevision(id, revision)) return; if (stillConfigured.has(pid)) continue; this.network.pruneConfiguredBootstrapPeer(pid); if (stillJoinedPeers.has(pid)) continue; @@ -279,6 +365,7 @@ export class Networks { // Network.disconnectPeer entry point (which also clears the keep-alive tag // so ReconnectQueue does not immediately re-dial it). for (const pid of leftPeers) { + if (!this.isCurrentRevision(id, revision)) return; if (stillJoinedPeers.has(pid)) continue; if (this.network.isBootstrapOrRelayPeer(pid)) continue; await this.network.disconnectPeer(pid, id); @@ -286,10 +373,6 @@ export class Networks { const net = this.get(id); console.log(`✓ Left lishnet: ${net?.name ?? id}`); - - // Notify higher layers (e.g. transfer) so downloads bound exclusively to - // this lishnet can be stopped. - this._onNetworkLeft?.(id); } /** diff --git a/backend/tests/unit/lishnet/enable-serialisation.test.ts b/backend/tests/unit/lishnet/enable-serialisation.test.ts new file mode 100644 index 000000000..7c38a7ab2 --- /dev/null +++ b/backend/tests/unit/lishnet/enable-serialisation.test.ts @@ -0,0 +1,176 @@ +import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; +import { initLISHnetsTables, addLISHnet, getLISHnet } from '../../../src/db/lishnets.ts'; +import { Networks } from '../../../src/lishnet/lishnets.ts'; + +/** + * Enable and disable of one lishnet must produce the state the LAST request asked + * for — in the database, in the pubsub subscription and in the callbacks the + * transfer layer listens to. + * + * Neither operation used to be serialised, and both await for a long time: a join + * waits on bootstrap dials, a leave disconnects peers one at a time. So an enable + * could announce a join after a disable had already left, and a leave could keep + * disconnecting the peers of a network that had just been re-enabled. + */ + +const NET = 'net-a'; +const BOOTSTRAP = '/ip4/192.0.2.1/tcp/9090/p2p/12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +function makeMockNet() { + return { + subscribed: [] as string[], + unsubscribed: [] as string[], + disconnected: [] as string[], + /** Set to hold the next bootstrap dial / peer disconnect open. */ + dialGate: null as null | Promise, + disconnectGate: null as null | Promise, + topicPeers: new Map(), + getTopicPeers(id: string): string[] { + return this.topicPeers.get(id) ?? []; + }, + getRecentTopicMembers(): string[] { + return []; + }, + subscribeTopic(id: string): void { + this.subscribed.push(id); + }, + unsubscribeTopic(id: string): void { + this.unsubscribed.push(id); + this.topicPeers.delete(id); + }, + isBootstrapOrRelayPeer(): boolean { + return false; + }, + async disconnectPeer(pid: string): Promise { + if (this.disconnectGate) await this.disconnectGate; + this.disconnected.push(pid); + }, + pruneConfiguredBootstrapPeer(): void {}, + resetBootstrapStatus(): void {}, + pruneBootstrapAddresses(): void {}, + pruneBootstrapStatus(): void {}, + clearRedialSuppressionForNetwork(): void {}, + async addBootstrapPeers(): Promise { + if (this.dialGate) await this.dialGate; + }, + }; +} + +function makeNetworks(net: ReturnType, db: Database, joined: string[]) { + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).db = db; + (networks as any).network = net; + (networks as any).joinedNetworks = new Set(joined); + (networks as any).networkOperations = new Map(); + (networks as any).desiredRevisions = new Map(); + (networks as any).announcedJoined = new Map(joined.map(id => [id, true])); + const events: string[] = []; + (networks as any)._onNetworkJoined = (id: string): void => { + events.push(`joined:${id}`); + }; + (networks as any)._onNetworkLeft = (id: string): void => { + events.push(`left:${id}`); + }; + return { networks, events }; +} + +describe('Networks.setEnabled — serialised per lishnet', () => { + let db: Database; + let net: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: false, created: new Date().toISOString() }); + net = makeMockNet(); + }); + + it('an enable overtaken by a disable never announces the join', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks, events } = makeNetworks(net, db, []); + + const enabling = networks.setEnabled(NET, true); + await Promise.resolve(); + // The user changes their mind while the bootstrap dial is still outstanding. + const disabling = networks.setEnabled(NET, false); + gate.resolve(); + await Promise.all([enabling, disabling]); + + expect(events).toEqual([]); + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + expect(net.unsubscribed).toEqual([NET]); + }); + + it('a disable overtaken by an enable never announces the leave', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks, events } = makeNetworks(net, db, [NET]); + + const disabling = networks.setEnabled(NET, false); + await Promise.resolve(); + const enabling = networks.setEnabled(NET, true); + gate.resolve(); + await Promise.all([disabling, enabling]); + + // From the outside nothing changed: the network never stopped being joined. + expect(events).toEqual([]); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + // The disconnect already in flight when the re-enable arrived cannot be recalled, + // but everything the leave had not reached yet belongs to the network we are back + // in and must be left alone. + expect(net.disconnected).not.toContain('p-only-a'); + }); + + it('three fast toggles land on the state the last one asked for', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks, events } = makeNetworks(net, db, []); + + const first = networks.setEnabled(NET, true); + await Promise.resolve(); + const second = networks.setEnabled(NET, false); + const third = networks.setEnabled(NET, true); + gate.resolve(); + await Promise.all([first, second, third]); + + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + expect(events).toEqual(['joined:' + NET]); + expect(net.unsubscribed).toEqual([]); + }); + + it('an uncontested enable still joins and announces it', async () => { + const { networks, events } = makeNetworks(net, db, []); + + await networks.setEnabled(NET, true); + + expect(events).toEqual(['joined:' + NET]); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect(net.subscribed).toEqual([NET]); + }); + + it('an uncontested disable still leaves and announces it', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const { networks, events } = makeNetworks(net, db, [NET]); + + await networks.setEnabled(NET, false); + + expect(events).toEqual(['left:' + NET]); + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect(net.disconnected).toContain('p-only-a'); + }); +}); diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 4087de2ce..501c3dac0 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -104,14 +104,19 @@ function makeNetworks(net: MockNet, joined: string[], configs: Record [id, true])); (networks as any)._onNetworkLeft = null; (networks as any)._onNetworkJoined = null; (networks as any).get = (id: string) => (configs[id] ? { networkID: id, bootstrapPeers: configs[id] } : undefined); return networks; } -const leave = (networks: Networks, id: string): Promise => (networks as any).leaveNetwork(id); -const join = (networks: Networks, id: string): Promise => (networks as any).joinNetwork(id); +// Both go through reconcile(), which is where the join/leave notifications live. +const leave = (networks: Networks, id: string): Promise => (networks as any).reconcile(id, false); +const join = (networks: Networks, id: string): Promise => (networks as any).reconcile(id, true); describe('Networks.leaveNetwork — exclusive peer disconnect', () => { let net: MockNet; From d6601791d89acb58e5ce911703ec7d88436c6377 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:02:02 +0200 Subject: [PATCH 90/99] fix(network): give each run its own in-flight dial claim set --- backend/src/protocol/network.ts | 21 +++++++--- .../protocol/network-restart-safety.test.ts | 39 +++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 6b67bff53..1cef97b10 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -409,8 +409,13 @@ export class Network { * and the peer-level backoff cannot help — it is only written once a dial has already * failed. Claiming the address for the duration of the dial makes the second run a * no-op instead: the first one is about to record the outcome both were after. + * + * Replaced with a FRESH Set on teardown rather than cleared, and captured by reference + * for the length of a dial. Clearing a shared Set let a dial still settling on the old + * node release a claim the new node's dial of the same address had just taken — after + * which a third request saw the address free and duplicated the live dial. */ - private readonly inFlightBootstrapDials = new Set(); + private inFlightBootstrapDials = new Set(); /** * peerID → time we first saw the peer disconnected with ZERO reachable * addresses. Such peers never enter the re-dial path (nothing to dial), so @@ -1639,6 +1644,9 @@ export class Network { // sweep until restart. Exactly the resurrection this eviction work exists to // prevent. const epoch = this.runEpoch; + // Captured, not re-read: the claims this run takes have to be released back into + // the same object it took them from, whatever teardown has since put in the field. + const inFlight = this.inFlightBootstrapDials; const generation = this.bootstrapGenerationOf(networkID); const superseded = (): boolean => epoch !== this.runEpoch || generation !== this.bootstrapGenerationOf(networkID); for (const peer of peers) { @@ -1740,11 +1748,11 @@ export class Network { // two runs asking about the SAME address duplicate a 10 s timeout for one // answer. Claimed after every skip above so a refused candidate never blocks // the run that would actually dial it. - if (this.inFlightBootstrapDials.has(canonicalAddress)) { + if (inFlight.has(canonicalAddress)) { trace(`[NET] addBootstrapPeers skip in-flight: ${peer}`); continue; } - this.inFlightBootstrapDials.add(canonicalAddress); + inFlight.add(canonicalAddress); // A CONFIGURED identity is user data and enters the set on the strength of the // saved config alone. A DISCOVERED one waits for the dial: it arrived in a // gossip message and nothing has yet shown that the identity exists, let @@ -1918,7 +1926,7 @@ export class Network { // Every exit from the dial block releases the claim, `return` included — // a leave landing mid-dial would otherwise lock the address out for the // lifetime of the node. - this.inFlightBootstrapDials.delete(canonicalAddress); + inFlight.delete(canonicalAddress); } } catch (error: any) { this.bootstrapTracker.recordOutcome(networkID, peer, null, 'error', error?.message ?? String(error), null, origin); @@ -2809,8 +2817,9 @@ export class Network { this.bootstrapMultiaddrs = []; this.bootstrapGeneration.clear(); // Claims belong to the node being torn down; a dial still settling on the old node - // must not lock the address out for the next one. - this.inFlightBootstrapDials.clear(); + // must not lock the address out for the next one — nor, by releasing into a shared + // Set, steal the claim the next one has taken. A new object does both. + this.inFlightBootstrapDials = new Set(); this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); diff --git a/backend/tests/unit/protocol/network-restart-safety.test.ts b/backend/tests/unit/protocol/network-restart-safety.test.ts index a87566019..32cd0d2de 100644 --- a/backend/tests/unit/protocol/network-restart-safety.test.ts +++ b/backend/tests/unit/protocol/network-restart-safety.test.ts @@ -156,6 +156,45 @@ describe('Network.addBootstrapPeers — a dial that lands after a restart', () = expect(disconnected).toEqual([]); }); + it('a dial settling after a restart does not release the claim of the new run', async () => { + const { network } = harness(); + let dials = 0; + const gates: Array<{ promise: Promise<{ remoteAddr: string }>; resolve: (v: { remoteAddr: string }) => void }> = []; + (network as any).node.dial = (): Promise<{ remoteAddr: string }> => { + dials++; + let resolve!: (v: { remoteAddr: string }) => void; + const promise = new Promise<{ remoteAddr: string }>(res => { + resolve = res; + }); + gates.push({ promise, resolve }); + return promise; + }; + + // Run A claims the address on the old node. + const runA = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(1); + + // Teardown hands the next run a fresh claim table; run B takes the address in it. + (network as any).inFlightBootstrapDials = new Set(); + (network as any).runEpoch = 2; + const runB = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(2); + + // Run A finally settles. Its release must land in the table it claimed from. + gates[0]!.resolve({ remoteAddr: ADDR }); + await runA; + + // A third request for the same address must still find run B's claim in place. + const runC = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(2); + + gates[1]!.resolve({ remoteAddr: ADDR }); + await Promise.all([runB, runC]); + }); + it('still disconnects when the dial lands on the same run after a leave', async () => { const { network, gate, disconnected } = harness(); // A leave bumps the generation; that must NOT short-circuit ahead of the From b2214e2d54cad0984c3d0c4e736a69a777011aa6 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:05:37 +0200 Subject: [PATCH 91/99] fix(peer-announce): tie the emitter loop to its start generation --- backend/src/protocol/peer-announce.ts | 42 +++++++--- .../tests/unit/protocol/peer-announce.test.ts | 78 +++++++++++++++++++ 2 files changed, 111 insertions(+), 9 deletions(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index 504e08843..f80f0c84c 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -180,6 +180,16 @@ export class PeerAnnounceManager { private readonly deps: PeerAnnounceManagerDeps; private timer: NodeJS.Timeout | null = null; private stopped = false; + /** + * Which start() a tick belongs to, bumped by both start() and stop(). + * + * The `stopped` flag alone cannot tell a tick apart from its successor: a loop parked + * in `await peerStore.all()` across a stop() AND a start() found the flag false again + * and carried on, so two loops armed timers into one `timer` field and the next stop() + * cancelled only the last of them. A generation is not reusable, so the parked loop + * ends where it stands. + */ + private generation = 0; /** * Per-topic recently-seen subscribers (peerID → last-seen ms). Lets a * momentarily-disconnected same-network peer stay an eligible transitive-announce @@ -255,7 +265,8 @@ export class PeerAnnounceManager { /** Start the periodic emitter. Safe to call only once per start/stop cycle. */ start(): void { this.stopped = false; - this.scheduleNext().catch(() => { + const generation = ++this.generation; + this.scheduleNext(generation).catch(() => { /* first-tick scheduling failure would leave emitter stopped — acceptable fallback */ }); } @@ -273,6 +284,7 @@ export class PeerAnnounceManager { */ stop(): void { this.stopped = true; + this.generation++; if (this.timer) { clearTimeout(this.timer); this.timer = null; @@ -371,8 +383,9 @@ export class PeerAnnounceManager { await this.deps.addBootstrapPeers(filtered, networkID, 'discovered'); } - private async scheduleNext(): Promise { - if (this.stopped) return; + /** Arm the next announce tick for `generation`; see {@link generation}. */ + private async scheduleNext(generation: number): Promise { + if (this.isSuperseded(generation)) return; const node = this.deps.getNode(); const pubsub = this.deps.getPubsub(); if (!node || !pubsub) return; @@ -386,26 +399,33 @@ export class PeerAnnounceManager { } catch { base = PEER_ANNOUNCE_INTERVAL_STEADY_MS; } - if (this.stopped) return; + // Checked before the timer is armed, not only before the work: a superseded loop + // that still armed one would leave two timers behind a single `timer` field. + if (this.isSuperseded(generation)) return; const jitter = Math.floor((Math.random() * 2 - 1) * base * PEER_ANNOUNCE_JITTER_RATIO); const delay = Math.max(5_000, base + jitter); this.timer = setTimeout(async () => { // Guard: stop() may have been called while we were sleeping. - if (this.stopped) return; + if (this.isSuperseded(generation)) return; try { - await this.emit(); + await this.emit(generation); } catch (err: any) { trace(`[NET] peer-announce emit error: ${err?.message ?? err}`); } // Guard again before scheduling the next tick. - if (this.stopped) return; - this.scheduleNext().catch(() => { + if (this.isSuperseded(generation)) return; + this.scheduleNext(generation).catch(() => { /* schedule is async but errors handled inline */ }); }, delay); } - private async emit(): Promise { + /** True once this tick's start() has been superseded by a stop() or a newer start(). */ + private isSuperseded(generation: number): boolean { + return this.stopped || generation !== this.generation; + } + + private async emit(generation: number = this.generation): Promise { const node = this.deps.getNode(); const pubsub = this.deps.getPubsub(); if (!node || !pubsub) return; @@ -415,6 +435,10 @@ export class PeerAnnounceManager { // threshold below. Broadcasting is what the threshold gates, not tracking. this.refreshMembers(lishTopics, pubsub); const allPeers = await node.peerStore.all(); + // The node and pubsub captured above belong to the run this tick started in; a + // restart landing in that await would otherwise have us publish the previous run's + // addresses onto the new node's topics. + if (this.isSuperseded(generation)) return; if (allPeers.length < PEER_ANNOUNCE_MIN_PEER_STORE) return; if (lishTopics.length === 0) return; const localCidrs = getLocalCidrs(); diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 6c777e536..10d71ca37 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -470,3 +470,81 @@ describe('PeerAnnounceManager.handle raw input bound', () => { expect(forwarded).toEqual([[named]]); }); }); + +/** + * The emitter's lifecycle is a boolean plus a single timer handle, and scheduleNext() + * awaits peerStore.all() before arming that timer. A loop parked in that await across + * a stop() AND a start() used to find the flag reset and carry on, so two loops armed + * timers into one field and the next stop() cancelled only one of them. + */ +describe('PeerAnnounceManager lifecycle — one loop per start', () => { + function gatedNode(gate: Promise) { + return { + peerId: { toString: (): string => SELF_ID }, + getMultiaddrs: (): unknown[] => [Multiaddr(SELF_ADDR)], + peerStore: { + all: async (): Promise => { + await gate; + return peersWithFillers(); + }, + }, + }; + } + + it('a loop parked across a stop/start does not arm a second timer', async () => { + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [] }; + const { mgr } = buildManager(gatedNode(gate), pubsub); + + const realSetTimeout = globalThis.setTimeout; + let armed = 0; + globalThis.setTimeout = ((fn: () => void, ms?: number) => { + armed++; + return realSetTimeout(fn, ms); + }) as typeof globalThis.setTimeout; + try { + mgr.start(); // loop A parks in peerStore.all() + await Promise.resolve(); + mgr.stop(); + mgr.start(); // loop B parks in its own peerStore.all() + release(); + // Let both awaits settle. + for (let i = 0; i < 8; i++) await Promise.resolve(); + expect(armed).toBe(1); + } finally { + globalThis.setTimeout = realSetTimeout; + mgr.stop(); + } + }); + + it('an emit whose run ended mid-await publishes nothing', async () => { + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + const { mgr, broadcasts } = buildManager(gatedNode(gate), pubsub); + + mgr.start(); + const emitting = (mgr as any).emit((mgr as any).generation); + mgr.stop(); + release(); + await emitting; + + expect(broadcasts).toEqual([]); + }); + + it('an uninterrupted emit still publishes', async () => { + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + const { mgr, broadcasts } = buildManager(gatedNode(Promise.resolve()), pubsub); + + mgr.start(); + await (mgr as any).emit((mgr as any).generation); + mgr.stop(); + + expect(broadcasts).toHaveLength(1); + }); +}); From b361b533c64c2d9658bb313a95468b7d17951246 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:07:05 +0200 Subject: [PATCH 92/99] fix(peer-announce): never advertise an address of a foreign identity --- backend/src/protocol/peer-announce.ts | 12 +++++- .../tests/unit/protocol/peer-announce.test.ts | 43 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index f80f0c84c..9f6945010 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -497,7 +497,17 @@ export class PeerAnnounceManager { skippedTransitive++; continue; } - const full = base.includes('/p2p/') ? base : `${base}/p2p/${pid}`; + // "Contains a /p2p/" is not the same question as "ends at THIS peer". A + // stale or poisoned peerStore address of A that terminates in /p2p/B was + // treated as already identified and broadcast verbatim, so every receiver + // learned it as B's address and dialed the wrong identity. Ask the + // destination — the same way the dial paths do — and skip what disagrees. + const destination = extractDestinationPeerID(addr.multiaddr); + if (destination !== null && destination !== pid) { + trace(`[NET] peer-announce skipping addr of ${pid.slice(0, 16)} that resolves to ${destination.slice(0, 16)}: ${base}`); + continue; + } + const full = destination === pid ? base : `${base}/p2p/${pid}`; if (!collected.has(full)) transitiveAdded++; collected.add(full); perPeer++; diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 10d71ca37..a557bca23 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -548,3 +548,46 @@ describe('PeerAnnounceManager lifecycle — one loop per start', () => { expect(broadcasts).toHaveLength(1); }); }); + +/** + * A peerStore entry for A can hold an address that terminates in /p2p/B — stale config, + * a poisoned announce, or a peer that changed identity. Broadcasting it verbatim taught + * every receiver that the address belongs to B. + */ +describe('PeerAnnounceManager.emit — transitive addresses must name their own peer', () => { + function emitWith(addresses: string[]) { + const peer = { id: { toString: (): string => PA_ID }, addresses: addresses.map(a => ({ multiaddr: Multiaddr(a) })) }; + const allPeers = peersWithFillers(peer as any); + const node = { + peerId: { toString: (): string => SELF_ID }, + getMultiaddrs: (): unknown[] => [Multiaddr(SELF_ADDR)], + peerStore: { all: async (): Promise => allPeers }, + }; + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + return buildManager(node, pubsub); + } + + it('drops an address of one peer that ends in another peer identity', async () => { + const { mgr, broadcasts } = emitWith([`${PA_ADDR}/p2p/${PB_ID}`]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toEqual([SELF_ADDR]); + }); + + it('keeps an address that already ends in the right identity', async () => { + const { mgr, broadcasts } = emitWith([`${PA_ADDR}/p2p/${PA_ID}`]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toContain(`${PA_ADDR}/p2p/${PA_ID}`); + }); + + it('appends the identity to a bare address', async () => { + const { mgr, broadcasts } = emitWith([PA_ADDR]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toContain(`${PA_ADDR}/p2p/${PA_ID}`); + }); +}); From d4c6224b15959b108c29a4294981c7d87611307d Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:15:48 +0200 Subject: [PATCH 93/99] fix(lishnets): reconcile the runtime after every stored write --- backend/src/api/lishnets.ts | 10 +- backend/src/lishnet/lishnets.ts | 65 ++++++-- .../unit/lishnet/import-reconcile.test.ts | 144 ++++++++++++++++++ .../tests/unit/lishnet/leave-network.test.ts | 65 +++++--- 4 files changed, 244 insertions(+), 40 deletions(-) create mode 100644 backend/tests/unit/lishnet/import-reconcile.test.ts diff --git a/backend/src/api/lishnets.ts b/backend/src/api/lishnets.ts index eb7113d01..221a6d1ff 100644 --- a/backend/src/api/lishnets.ts +++ b/backend/src/api/lishnets.ts @@ -10,11 +10,11 @@ interface LISHnetsHandlers { get: (p: { networkID: string }) => LISHNetworkConfig | undefined; exists: (p: { networkID: string }) => boolean; add: (p: { network: LISHNetworkConfig }) => Promise; - update: (p: { network: LISHNetworkConfig }) => boolean; + update: (p: { network: LISHNetworkConfig }) => Promise; delete: (p: { networkID: string }) => Promise; addIfNotExists: (p: { network: LISHNetworkDefinition }) => Promise; import: (p: { networks: LISHNetworkDefinition[] }) => number; - replace: (p: { networks: LISHNetworkConfig[] }) => boolean; + replace: (p: { networks: LISHNetworkConfig[] }) => Promise; exportToFile: (p: { networkID: string; filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }) => Promise; exportAllToFile: (p: { filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }) => Promise; importFromFile: (p: { path: string; enabled?: boolean }) => Promise; @@ -54,7 +54,7 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, assert(p, ['network']); return networks.add(p.network); } - function update(p: { network: LISHNetworkConfig }): boolean { + async function update(p: { network: LISHNetworkConfig }): Promise { assert(p, ['network']); return networks.update(p.network); } @@ -70,9 +70,9 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, assert(p, ['networks']); return networks.importNetworks(p.networks); } - function replace(p: { networks: LISHNetworkConfig[] }): boolean { + async function replace(p: { networks: LISHNetworkConfig[] }): Promise { assert(p, ['networks']); - networks.replace(p.networks); + await networks.replace(p.networks); return true; } async function exportToFile(p: { networkID: string; filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }): Promise { diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index b2f1cc257..af06d0b4e 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -196,6 +196,31 @@ export class Networks { }); } + /** + * Bring the runtime in line with what the database now says about one lishnet. + * + * Every writer used to be responsible for this itself, and most of them simply were + * not: importing an already-joined network rewrote its bootstrap list in the database + * while the node went on dialing the old one, and importing an active network as + * disabled left it joined until the next restart. The decision is made from the + * RUNTIME state rather than the previous row, because that is what has to change. + * + * `previous` is the row as it was before the write, and is needed only to tell whether + * the bootstrap list moved. + */ + private async reconcileStored(id: string, previous: LISHNetworkConfig | undefined): Promise { + const next = this.get(id); + const wantJoined = next?.enabled === true; + const joined = this.joinedNetworks.has(id); + const before = Networks.cleanBootstrapList(previous?.bootstrapPeers ?? []); + const after = Networks.cleanBootstrapList(next?.bootstrapPeers ?? []); + // A list change is picked up first so a network that is about to be joined is + // joined against the pruned state — except when we are on our way OUT of it, where + // the leave resets the whole status anyway and a dial would be pure waste. + if (!(joined && !wantJoined) && before.join('\n') !== after.join('\n')) this.syncBootstrapRuntime(id, previous?.bootstrapPeers ?? [], after); + if (joined !== wantJoined) await this.reconcile(id, wantJoined); + } + /** True while `revision` is still the newest request for this lishnet. */ private isCurrentRevision(id: string, revision: number | undefined): boolean { return revision === undefined || this.desiredRevisions.get(id) === revision; @@ -449,8 +474,9 @@ export class Networks { async importFromLISHnet(data: ILISHNetwork, enabled: boolean = false): Promise { const definition = this.validateNetwork(data); const config: LISHNetworkConfig = { ...definition, enabled }; + const previous = this.get(config.networkID); upsertLISHnet(this.db, config.networkID, config.name, config.description, config.bootstrapPeers, config.enabled, config.created); - if (enabled) await this.joinNetwork(config.networkID); + await this.reconcileStored(config.networkID, previous); return config; } @@ -490,24 +516,26 @@ export class Networks { return listEnabledLISHnets(this.db); } - add(network: LISHNetworkConfig): boolean { - return addLISHnet(this.db, network); + async add(network: LISHNetworkConfig): Promise { + const ok = addLISHnet(this.db, network); + // A network added as enabled has to be joined, not merely written down. + if (ok) await this.reconcileStored(network.networkID, undefined); + return ok; } - update(network: LISHNetworkConfig): boolean { + async update(network: LISHNetworkConfig): Promise { const existing = this.get(network.networkID); // Store the cleaned list, not the raw one: blank rows from the form would // otherwise be persisted while the runtime worked from the filtered copy, and // the two would disagree about what this network's bootstrap list even is. const cleaned = Networks.cleanBootstrapList(network.bootstrapPeers ?? []); const ok = updateLISHnet(this.db, { ...network, bootstrapPeers: cleaned }); - // The general edit form carries the bootstrap list as well, so this path can - // change it just like updateBootstrapPeers does. Without the same runtime - // synchronisation the edit would reach only the database and the live node - // would keep dialing the previous list until restart. - if (!ok || !existing) return ok; - const previous = Networks.cleanBootstrapList(existing.bootstrapPeers); - if (previous.join('\n') !== cleaned.join('\n')) this.syncBootstrapRuntime(network.networkID, existing.bootstrapPeers, cleaned); + // The general edit form carries the bootstrap list AND the enabled flag, so this + // path can change either one. Without the runtime reconciliation the edit would + // reach only the database and the live node would keep dialing the previous list — + // or stay in a network the edit had just disabled — until restart. + if (!ok) return ok; + await this.reconcileStored(network.networkID, existing); return ok; } @@ -524,12 +552,25 @@ export class Networks { return addLISHnetIfNotExists(this.db, network); } + /** + * Add every definition that does not exist yet. Nothing to reconcile: the underlying + * writer skips networks that already exist and inserts new ones DISABLED, so no + * network's runtime state can change here. + */ importNetworks(networks: LISHNetworkDefinition[]): number { return importLISHnets(this.db, networks); } - replace(networks: LISHNetworkConfig[]): void { + /** + * Replace the whole stored list (used for reordering). Every network the write touched + * is reconciled afterwards, including ones it dropped: a wholesale rewrite can enable, + * disable, re-list or delete anything, and a deleted network that is still joined would + * otherwise stay in its topic with no row left to explain it. + */ + async replace(networks: LISHNetworkConfig[]): Promise { + const before = new Map(this.list().map(n => [n.networkID, n])); replaceLISHnets(this.db, networks); + for (const id of new Set([...before.keys(), ...networks.map(n => n.networkID)])) await this.reconcileStored(id, before.get(id)); } /** diff --git a/backend/tests/unit/lishnet/import-reconcile.test.ts b/backend/tests/unit/lishnet/import-reconcile.test.ts new file mode 100644 index 000000000..ef67276d3 --- /dev/null +++ b/backend/tests/unit/lishnet/import-reconcile.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { initLISHnetsTables, addLISHnet, getLISHnet } from '../../../src/db/lishnets.ts'; +import { Networks } from '../../../src/lishnet/lishnets.ts'; + +/** + * Importing a network writes the database. It must also reach the running node: + * importing an already-joined network used to leave the live bootstrap list, statuses + * and autodial addresses on the previous configuration, and importing an active + * network as disabled left it joined until the next restart. + */ + +const NET = 'net-a'; +const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; +const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; +const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + +function makeMockNet() { + return { + subscribed: [] as string[], + unsubscribed: [] as string[], + dialledLists: [] as Array<{ networkID: string; peers: string[]; origin: string }>, + prunedStatus: [] as Array<{ networkID: string; keep: string[] }>, + prunedAddresses: [] as string[][], + prunedBootstrap: [] as string[], + getTopicPeers: (): string[] => [], + getRecentTopicMembers: (): string[] => [], + isBootstrapOrRelayPeer: (): boolean => false, + async disconnectPeer(): Promise {}, + pruneConfiguredBootstrapPeer(pid: string): void { + this.prunedBootstrap.push(pid); + }, + resetBootstrapStatus(): void {}, + pruneBootstrapAddresses(addresses: string[]): void { + this.prunedAddresses.push(addresses); + }, + pruneBootstrapStatus(networkID: string, keep: string[]): void { + this.prunedStatus.push({ networkID, keep }); + }, + clearRedialSuppressionForNetwork(): void {}, + subscribeTopic(id: string): void { + this.subscribed.push(id); + }, + unsubscribeTopic(id: string): void { + this.unsubscribed.push(id); + }, + async addBootstrapPeers(peers: string[], networkID: string, origin: string): Promise { + this.dialledLists.push({ networkID, peers, origin }); + }, + }; +} + +function bare(db: Database, mock: ReturnType, joined: string[]) { + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).db = db; + (networks as any).network = mock; + (networks as any).joinedNetworks = new Set(joined); + (networks as any).networkOperations = new Map(); + (networks as any).desiredRevisions = new Map(); + (networks as any).announcedJoined = new Map(joined.map(id => [id, true])); + return networks; +} + +describe('Networks.importFromLISHnet — the runtime follows the import', () => { + let db: Database; + let mock: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + mock = makeMockNet(); + }); + + it('switches a joined network over to the imported bootstrap list', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, [NET]); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_B] } as any, true); + + expect(getLISHnet(db, NET)!.bootstrapPeers).toEqual([ADDR_B]); + expect(mock.prunedStatus).toEqual([{ networkID: NET, keep: [ADDR_B] }]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_B], origin: 'configured' }]); + }); + + it('leaves an active network imported as disabled', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, [NET]); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, false); + + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect(mock.unsubscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('joins a brand-new network imported as enabled', async () => { + const networks = bare(db, mock, []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, true); + + expect(mock.subscribed).toEqual([NET]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_A], origin: 'configured' }]); + }); + + it('touches nothing at runtime for a disabled network imported as disabled', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: false, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, false); + + expect(mock.subscribed).toEqual([]); + expect(mock.unsubscribed).toEqual([]); + expect(mock.dialledLists).toEqual([]); + }); +}); + +describe('Networks.replace — a wholesale rewrite reaches the runtime', () => { + it('leaves a joined network the rewrite dropped', async () => { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = bare(db, mock, [NET]); + + await networks.replace([]); + + expect(mock.unsubscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('keeps a re-listed network joined', async () => { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = bare(db, mock, [NET]); + + await networks.replace([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }]); + + expect(mock.unsubscribed).toEqual([]); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + }); +}); diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 501c3dac0..c92850c37 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -365,43 +365,46 @@ describe('Networks.joinNetwork — onNetworkJoined notification', () => { * the form was written to the database and then ignored until restart — the node * kept dialing the peers the user had just removed. */ -describe('Networks.update — a changed bootstrap list reaches the running node', () => { +describe('Networks.update — a changed bootstrap list reaches the running node', async () => { const NET = 'net-a'; const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; - function seeded(bootstrapPeers: string[]) { + function seeded(bootstrapPeers: string[], enabled = true) { const db = new Database(':memory:'); initLISHnetsTables(db); - addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers, enabled: true, created: '2026-01-01T00:00:00.000Z' }); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers, enabled, created: '2026-01-01T00:00:00.000Z' }); const mock = makeMockNet(); const networks = Object.create(Networks.prototype) as Networks; (networks as any).network = mock; (networks as any).db = db; - (networks as any).joinedNetworks = new Set([NET]); + (networks as any).joinedNetworks = new Set(enabled ? [NET] : []); + (networks as any).networkOperations = new Map(); + (networks as any).desiredRevisions = new Map(); + (networks as any).announcedJoined = new Map(enabled ? [[NET, true]] : []); return { networks, mock, db }; } - const edit = (networks: Networks, bootstrapPeers: string[]): boolean => (networks as any).update({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const edit = (networks: Networks, bootstrapPeers: string[], enabled = true): Promise => (networks as any).update({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled, created: '2026-01-01T00:00:00.000Z' }); - it('prunes the status and dials the new list when the entries change', () => { + it('prunes the status and dials the new list when the entries change', async () => { const { networks, mock } = seeded([ADDR_A]); - edit(networks, [ADDR_B]); + await edit(networks, [ADDR_B]); expect(mock.prunedStatus).toEqual([{ networkID: NET, keep: [ADDR_B] }]); expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_B], origin: 'configured' }]); }); - it('drops the bootstrap exemption of a peer removed through the form', () => { + it('drops the bootstrap exemption of a peer removed through the form', async () => { const { networks, mock } = seeded([ADDR_A, ADDR_B]); - edit(networks, [ADDR_A]); + await edit(networks, [ADDR_A]); expect(mock.prunedBootstrap).toEqual([PEER_B]); }); - it('leaves the running node alone when only the name changed', () => { + it('leaves the running node alone when only the name changed', async () => { const { networks, mock } = seeded([ADDR_A]); - edit(networks, [ADDR_A]); + await edit(networks, [ADDR_A]); expect(mock.prunedStatus).toEqual([]); expect(mock.dialledLists).toEqual([]); expect(mock.prunedBootstrap).toEqual([]); @@ -412,9 +415,9 @@ describe('Networks.update — a changed bootstrap list reaches the running node' * the filtered copy left the database and the live node disagreeing about what the * network's bootstrap list actually is. */ - it('persists the cleaned list, not the blank rows the form submitted', () => { + it('persists the cleaned list, not the blank rows the form submitted', async () => { const { networks, db } = seeded([ADDR_A]); - edit(networks, ['', ADDR_B, ' ']); + await edit(networks, ['', ADDR_B, ' ']); expect(getLISHnet(db, NET)?.bootstrapPeers).toEqual([ADDR_B]); }); @@ -423,19 +426,19 @@ describe('Networks.update — a changed bootstrap list reaches the running node' * nothing to do. Without an address-level prune the replaced address stays on the * autodial list and recovery keeps dialing it. */ - it('drops the replaced address when only the host changed', () => { + it('drops the replaced address when only the host changed', async () => { const moved = `/ip4/203.0.113.99/tcp/9090/p2p/${PEER_A}`; const { networks, mock } = seeded([ADDR_A]); - edit(networks, [moved]); + await edit(networks, [moved]); expect(mock.prunedAddresses).toEqual([[ADDR_A]]); expect(mock.prunedBootstrap).toEqual([]); }); - it('keeps an address that is still configured for another joined network', () => { + it('keeps an address that is still configured for another joined network', async () => { const { networks, mock } = seeded([ADDR_A]); (networks as any).get = (nid: string) => (nid === 'net-other' ? { networkID: nid, bootstrapPeers: [ADDR_A] } : { networkID: NET, bootstrapPeers: [ADDR_A] }); (networks as any).joinedNetworks = new Set([NET, 'net-other']); - edit(networks, [ADDR_B]); + await edit(networks, [ADDR_B]); expect(mock.prunedAddresses).toEqual([[]]); }); @@ -444,19 +447,35 @@ describe('Networks.update — a changed bootstrap list reaches the running node' * the list" check has to as well — otherwise one spelling of an address counts as a * removal here and as the same entry there. */ - it('treats two spellings of one address as the same entry', () => { + it('treats two spellings of one address as the same entry', async () => { const upper = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PEER_A}`; const lower = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PEER_A}`; const { networks, mock } = seeded([upper]); - edit(networks, [lower]); + await edit(networks, [lower]); expect(mock.prunedAddresses).toEqual([[]]); }); - it('does not dial for a network that is not joined', () => { - const { networks, mock } = seeded([ADDR_A]); - (networks as any).joinedNetworks = new Set(); - edit(networks, [ADDR_B]); + it('does not dial for a network that is not joined', async () => { + const { networks, mock } = seeded([ADDR_A], false); + await edit(networks, [ADDR_B], false); expect(mock.dialledLists).toEqual([]); expect(mock.prunedStatus).toHaveLength(1); }); + + /** + * The edit form carries the enabled flag too, so it can turn a network on or off — + * and that has to reach the node, not just the database row. + */ + it('joins a network the edit enabled', async () => { + const { networks, mock } = seeded([ADDR_A], false); + await edit(networks, [ADDR_A], true); + expect(mock.subscribed).toEqual([NET]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_A], origin: 'configured' }]); + }); + + it('leaves a network the edit disabled', async () => { + const { networks, mock } = seeded([ADDR_A], true); + await edit(networks, [ADDR_A], false); + expect(mock.unsubscribed).toEqual([NET]); + }); }); From 5d7b2d6677650587e8733c990361db17559ae46b Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:19:02 +0200 Subject: [PATCH 94/99] fix(lishnets): normalise bootstrap lists at the write boundary --- backend/src/db/lishnets.ts | 42 ++++++++++- backend/src/lishnet/lishnets.ts | 31 +++----- .../unit/lishnet/import-reconcile.test.ts | 71 +++++++++++++++++++ 3 files changed, 118 insertions(+), 26 deletions(-) diff --git a/backend/src/db/lishnets.ts b/backend/src/db/lishnets.ts index 82124a135..030a69249 100644 --- a/backend/src/db/lishnets.ts +++ b/backend/src/db/lishnets.ts @@ -1,5 +1,36 @@ import { type Database } from 'bun:sqlite'; import { type LISHNetworkConfig, type LISHNetworkDefinition } from '@shared'; +import { canonicalMultiaddr } from '../protocol/multiaddr-utils.ts'; + +/** + * Normalise a bootstrap list: drop non-strings and blanks, trim, and keep one entry per + * canonical address. + * + * Trimming matters because the list usually came from a text field — a value with stray + * whitespace passed the old blank check and then failed to parse at dial time. + * Deduplicating matters because two spellings of one address (DNS case, trailing dot, + * expanded vs compressed IPv6) would otherwise each get their own forced probe and their + * own status row for the same endpoint. + * + * It lives HERE, at the write boundary, and is applied by every writer below. As a helper + * the callers were free to skip, only the two edit paths used it, so import, add and + * wholesale replace could all store values that the rest of the system then had to cope + * with. + */ +export function cleanBootstrapList(peers: string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of peers) { + if (typeof raw !== 'string') continue; + const trimmed = raw.trim(); + if (trimmed.length === 0) continue; + const key = canonicalMultiaddr(trimmed); + if (seen.has(key)) continue; + seen.add(key); + out.push(trimmed); + } + return out; +} export function initLISHnetsTables(db: Database): void { db.run(` @@ -32,6 +63,11 @@ function getInternalID(db: Database, networkID: string): number | null { return row?.id ?? null; } +/** The one place a bootstrap list is written — so the one place it is normalised. */ +function writeBootstrapPeers(db: Database, internalID: number, peers: string[]): void { + for (const peer of cleanBootstrapList(peers)) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); +} + function getBootstrapPeers(db: Database, internalID: number): string[] { return db .query<{ address: string }, [number]>('SELECT address FROM lishnets_peers WHERE id_lishnets = ? ORDER BY id') @@ -95,7 +131,7 @@ export function addLISHnet(db: Database, network: LISHNetworkConfig): boolean { ); const internalID = Number(result.lastInsertRowid); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); }); tx(); return true; @@ -114,7 +150,7 @@ export function updateLISHnet(db: Database, network: LISHNetworkConfig): boolean // Replace peers db.run('DELETE FROM lishnets_peers WHERE id_lishnets = ?', [internalID]); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); }); tx(); return true; @@ -163,7 +199,7 @@ export function replaceLISHnets(db: Database, networks: LISHNetworkConfig[]): vo [network.networkID, network.name, network.description || null, network.enabled ? 1 : 0, network.created || null] ); const internalID = Number(result.lastInsertRowid); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); } }); tx(); diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index af06d0b4e..36dfce06a 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -1,12 +1,11 @@ import { type Database } from 'bun:sqlite'; import { Mutex } from 'async-mutex'; import { Network, normalizeMultiaddrForCompare } from '../protocol/network.ts'; -import { canonicalMultiaddr } from '../protocol/multiaddr-utils.ts'; import { Utils } from '../utils.ts'; import { type DataServer } from '../lish/data-server.ts'; import { type Settings } from '../settings.ts'; import { type ILISHNetwork, type LISHNetworkConfig, type LISHNetworkDefinition, type PeerConnectionInfo, type IMeshHealth, type BootstrapStatus, CodedError, ErrorCodes } from '@shared'; -import { lishnetExists, getLISHnet, listLISHnets, listEnabledLISHnets, addLISHnet, updateLISHnet, deleteLISHnet, setLISHnetEnabled, addLISHnetIfNotExists, importLISHnets, upsertLISHnet, replaceLISHnets } from '../db/lishnets.ts'; +import { cleanBootstrapList, lishnetExists, getLISHnet, listLISHnets, listEnabledLISHnets, addLISHnet, updateLISHnet, deleteLISHnet, setLISHnetEnabled, addLISHnetIfNotExists, importLISHnets, upsertLISHnet, replaceLISHnets } from '../db/lishnets.ts'; /** * Manages lishnets (logical network groups) on top of a single shared Network (libp2p) node. @@ -466,7 +465,10 @@ export class Networks { networkID: data.networkID, name: data.name, description: data.description || '', - bootstrapPeers: Array.isArray(data.bootstrapPeers) ? data.bootstrapPeers.filter(p => typeof p === 'string' && p.trim()) : [], + // Cleaned here as well as on write: this shape is also handed straight back to the + // caller as an import preview, and a preview that still shows the untrimmed value + // describes something other than what would be stored. + bootstrapPeers: Array.isArray(data.bootstrapPeers) ? cleanBootstrapList(data.bootstrapPeers) : [], created: data.created || new Date().toISOString(), }; } @@ -607,28 +609,11 @@ export class Networks { } /** - * Normalise a user-supplied bootstrap list: drop blanks, trim, and keep one entry per - * canonical address. - * - * Trimming matters because the list came from a text field — a value with stray - * whitespace passed the old blank check and then failed to parse at dial time. - * Deduplicating matters because two spellings of one address (DNS case, expanded vs - * compressed IPv6) would otherwise each get their own forced probe and their own - * status row for the same endpoint. + * The same normalisation the repository applies on write, for the comparisons here + * that have to speak about a list in the shape it will be stored in. */ private static cleanBootstrapList(peers: string[]): string[] { - const seen = new Set(); - const out: string[] = []; - for (const raw of peers) { - if (typeof raw !== 'string') continue; - const trimmed = raw.trim(); - if (trimmed.length === 0) continue; - const key = canonicalMultiaddr(trimmed); - if (seen.has(key)) continue; - seen.add(key); - out.push(trimmed); - } - return out; + return cleanBootstrapList(peers); } /** diff --git a/backend/tests/unit/lishnet/import-reconcile.test.ts b/backend/tests/unit/lishnet/import-reconcile.test.ts index ef67276d3..8a89d090c 100644 --- a/backend/tests/unit/lishnet/import-reconcile.test.ts +++ b/backend/tests/unit/lishnet/import-reconcile.test.ts @@ -142,3 +142,74 @@ describe('Networks.replace — a wholesale rewrite reaches the runtime', () => { expect((networks as any).joinedNetworks.has(NET)).toBe(true); }); }); + +/** + * Normalisation belongs to the write, not to the caller. Only the two edit paths used to + * apply it, so import, add and wholesale replace could store whitespace-padded values + * that fail to parse at dial time, and two spellings of one endpoint that each earn their + * own probe and their own status row. + */ +describe('bootstrap lists are normalised by whoever writes them', () => { + const PADDED = ` ${ADDR_A} `; + const UPPER = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PEER_A}`; + const LOWER = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PEER_A}`; + + function db(): Database { + const d = new Database(':memory:'); + initLISHnetsTables(d); + return d; + } + + it('add() trims and drops blanks', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.add({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED, '', ' '], enabled: false, created: '' }); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('addIfNotExists() collapses two spellings of one address', () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + networks.addIfNotExists({ networkID: NET, name: 'A', description: '', bootstrapPeers: [UPPER, LOWER], created: '' }); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([UPPER]); + }); + + it('importNetworks() trims what it stores', () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + networks.importNetworks([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED], created: '' }]); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('importFromLISHnet() trims what it stores', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED] } as any, false); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('replace() trims what it stores', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.replace([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED], enabled: false, created: '' }]); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('validateNetwork() reports the list in the shape it would be stored in', () => { + const networks = bare(db(), makeMockNet(), []); + + const definition = networks.validateNetwork({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED, ' '] } as any); + + expect(definition.bootstrapPeers).toEqual([ADDR_A]); + }); +}); From 97a1a000980c612cef88c3f786a4f6f3295d8916 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:23:52 +0200 Subject: [PATCH 95/99] fix(bootstrap-status): key rows by the canonical multiaddr --- backend/src/protocol/bootstrap-status.ts | 51 +++++++++++----- .../unit/protocol/bootstrap-status.test.ts | 61 +++++++++++++++++++ 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 184c83ec4..7daaaff8b 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -35,11 +35,29 @@ const BATCH_FLUSH_INTERVAL_MS = 75; */ type TrackedPeer = BootstrapPeerStatus & { staleSince: number }; +/** + * Which spelling of an endpoint the UI should show. + * + * The row is keyed canonically, so several spellings can land on it. A configured one + * always wins — it is what the user typed into the form and what they will look for when + * fixing it — and otherwise the first spelling seen is kept, so a row does not visibly + * change address every time gossip restates it differently. + */ +function displaySpelling(previous: TrackedPeer | undefined, incoming: string, origin: BootstrapPeerOrigin): string { + if (!previous || origin === 'configured') return incoming; + return previous.multiaddr; +} + /** * Tracks per-network, per-bootstrap-peer dial outcome status. * - * Outer key is networkID; inner key is the exact multiaddr string from the network - * config. Populated by markBootstrapPending / recordBootstrapOutcome when called + * Outer key is networkID; inner key is the CANONICAL form of the multiaddr, while the + * row keeps the spelling as written for display. Keying by the raw string let two + * spellings of one endpoint — DNS case, a trailing dot, an expanded IPv6 literal — open + * two rows that then contradicted each other, spent the row budget twice and survived a + * delete aimed at only one of them. + * + * Populated by markBootstrapPending / recordBootstrapOutcome when called * with a networkID context. Lets the UI surface which SPECIFIC bootstrap entry is * stale (identity-mismatch) or unreachable (timeout), rather than flagging the * whole network. @@ -186,11 +204,13 @@ export class BootstrapStatusTracker { markPending(networkID: string | null, multiaddr: string, expectedPeerID: string | null, origin: BootstrapPeerOrigin): void { if (!networkID) return; const net = this.ensureNetwork(networkID); + const key = canonicalMultiaddr(multiaddr); // Preserve a stronger origin classification — once we know an entry is in // the saved config ('configured'), an inbound peer-announce later restating // the same multiaddr must not downgrade it to 'discovered'. - const previous = net.get(multiaddr); + const previous = net.get(key); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; + const display = displaySpelling(previous, multiaddr, origin); // Keep the existing staleness clock (see sweepStale). Reaching this point means // someone MENTIONED the peer again — gossip repeating an address it still // remembers — which is evidence about the announcer, not about the peer. Letting @@ -198,7 +218,7 @@ export class BootstrapStatusTracker { // is far shorter than the sweep TTL, so the row was refreshed long before it // could expire, no matter how many dials to it had already failed. Only a dial // that actually CONNECTED advances it, in recordOutcome below. - net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString(), staleSince: previous?.staleSince ?? Date.now() }); + net.set(key, { multiaddr: display, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString(), staleSince: previous?.staleSince ?? Date.now() }); this.capDiscovered(net); this.notify(networkID); } @@ -207,14 +227,16 @@ export class BootstrapStatusTracker { recordOutcome(networkID: string | null, multiaddr: string, expectedPeerID: string | null, status: BootstrapPeerDialStatus, message: string | null, actualPeerID: string | null, origin: BootstrapPeerOrigin): void { if (!networkID) return; const net = this.ensureNetwork(networkID); + const key = canonicalMultiaddr(multiaddr); const truncated = message ? (message.length > 200 ? message.slice(0, 200) + '…' : message) : null; - const previous = net.get(multiaddr); + const previous = net.get(key); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; + const display = displaySpelling(previous, multiaddr, origin); // Only success restarts the staleness clock. A FAILING outcome is the node's own // reaction to somebody else's mention of a dead peer, so letting it advance the // clock kept exactly the rows this sweep exists to remove: gossip mentions the // peer, the dial fails, the row is refreshed, and the TTL is never reached. - net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString(), staleSince: status === 'connected' ? Date.now() : (previous?.staleSince ?? Date.now()) }); + net.set(key, { multiaddr: display, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString(), staleSince: status === 'connected' ? Date.now() : (previous?.staleSince ?? Date.now()) }); this.capDiscovered(net); this.notify(networkID); } @@ -234,13 +256,10 @@ export class BootstrapStatusTracker { recordAddressReachable(address: string): void { const target = canonicalMultiaddr(address); for (const [networkID, peers] of this.stats) { - let changed = false; - for (const [addr, peer] of peers) { - if (peer.status === 'connected' || canonicalMultiaddr(addr) !== target) continue; - peers.set(addr, { ...peer, status: 'connected', lastError: null, actualPeerID: null, updatedAt: new Date().toISOString(), staleSince: Date.now() }); - changed = true; - } - if (changed) this.notify(networkID); + const peer = peers.get(target); + if (!peer || peer.status === 'connected') continue; + peers.set(target, { ...peer, status: 'connected', lastError: null, actualPeerID: null, updatedAt: new Date().toISOString(), staleSince: Date.now() }); + this.notify(networkID); } } @@ -257,7 +276,7 @@ export class BootstrapStatusTracker { deletePeer(networkID: string, multiaddr: string): void { const net = this.stats.get(networkID); if (!net) return; - net.delete(multiaddr); + net.delete(canonicalMultiaddr(multiaddr)); if (net.size === 0) this.stats.delete(networkID); this.notify(networkID); } @@ -324,7 +343,9 @@ export class BootstrapStatusTracker { pruneEntries(networkID: string, keepMultiaddrs: string[]): void { const peers = this.stats.get(networkID); if (!peers) return; - const keep = new Set(keepMultiaddrs); + // Canonical, like the keys themselves: a configured entry re-typed in a different + // but equivalent spelling is the same entry, not a removed one. + const keep = new Set(keepMultiaddrs.map(canonicalMultiaddr)); for (const [addr, peer] of [...peers.entries()]) { if (peer.origin === 'configured' && !keep.has(addr)) peers.delete(addr); } diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index a3324c845..528b9be65 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -588,3 +588,64 @@ describe('BootstrapStatusTracker.recordAddressReachable', () => { expect(seen).toEqual([]); }); }); + +/** + * Rows are keyed by the canonical form of the endpoint. Keying by the raw string let two + * spellings of one address — DNS case, a trailing dot, an expanded IPv6 literal — open + * two contradictory rows, spend the discovered budget twice, and survive a delete aimed + * at only one of them. + */ +describe('BootstrapStatusTracker — one row per endpoint, whatever the spelling', () => { + const PID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const UPPER = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PID}`; + const LOWER = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PID}`; + const NET = 'net-a'; + + it('folds two spellings into a single row', () => { + const tracker = new BootstrapStatusTracker(); + + tracker.recordOutcome(NET, UPPER, PID, 'error', 'boom', null, 'discovered'); + tracker.recordOutcome(NET, LOWER, PID, 'connected', null, null, 'discovered'); + + const peers = tracker.getStatus(NET)!.peers; + expect(peers).toHaveLength(1); + expect(peers[0]!.status).toBe('connected'); + }); + + it('keeps the first spelling for display, and lets a configured one replace it', () => { + const tracker = new BootstrapStatusTracker(); + + tracker.recordOutcome(NET, LOWER, PID, 'error', 'boom', null, 'discovered'); + expect(tracker.getStatus(NET)!.peers[0]!.multiaddr).toBe(LOWER); + + tracker.markPending(NET, UPPER, PID, 'configured'); + expect(tracker.getStatus(NET)!.peers[0]!.multiaddr).toBe(UPPER); + }); + + it('deletes the row whichever spelling names it', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'error', 'boom', null, 'discovered'); + + tracker.deletePeer(NET, UPPER); + + expect(tracker.getStatus(NET)).toBeNull(); + }); + + it('keeps a configured row that was re-typed in another spelling', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'connected', null, null, 'configured'); + + tracker.pruneEntries(NET, [UPPER]); + + expect(tracker.getStatus(NET)!.peers).toHaveLength(1); + }); + + it('marks the row reachable when the probe names another spelling', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'timeout', 'no answer', null, 'configured'); + + tracker.recordAddressReachable(UPPER); + + expect(tracker.getStatus(NET)!.peers[0]!.status).toBe('connected'); + }); +}); From 8e900c1fadd501397ea23ae981f40543e5ad3c64 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:27:44 +0200 Subject: [PATCH 96/99] fix(bootstrap-status): evict dead rows before live ones at the cap --- backend/src/protocol/bootstrap-status.ts | 46 +++++++++++++++--- backend/src/protocol/network.ts | 2 + .../unit/protocol/bootstrap-status.test.ts | 48 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index 7daaaff8b..c7d4c50cb 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -72,12 +72,24 @@ export class BootstrapStatusTracker { private onStatusChange: ((networkID: string, status: BootstrapStatus) => void) | null = null; /** Open {@link batch} frames, keyed by networkID. See that method for why. */ private readonly batches: Map = new Map(); + /** Current members of a network, for {@link capDiscovered}. See {@link setMembersProvider}. */ + private membersProvider: ((networkID: string) => Set) | null = null; /** Register a callback that fires on every status mutation. */ setOnChange(cb: ((networkID: string, status: BootstrapStatus) => void) | null): void { this.onStatusChange = cb; } + /** + * Supply the current member set of a network, so {@link capDiscovered} can tell a row + * that describes a live participant from one that describes an address nobody answers + * on. Asked for the whole set rather than per peer because the cap ranks every row at + * once and a per-peer question would rebuild the same snapshot for each of them. + */ + setMembersProvider(fn: ((networkID: string) => Set) | null): void { + this.membersProvider = fn; + } + /** * Group many mutations of one network into a SINGLE status emission. * @@ -219,7 +231,7 @@ export class BootstrapStatusTracker { // could expire, no matter how many dials to it had already failed. Only a dial // that actually CONNECTED advances it, in recordOutcome below. net.set(key, { multiaddr: display, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString(), staleSince: previous?.staleSince ?? Date.now() }); - this.capDiscovered(net); + this.capDiscovered(networkID, net); this.notify(networkID); } @@ -237,7 +249,7 @@ export class BootstrapStatusTracker { // clock kept exactly the rows this sweep exists to remove: gossip mentions the // peer, the dial fails, the row is refreshed, and the TTL is never reached. net.set(key, { multiaddr: display, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString(), staleSince: status === 'connected' ? Date.now() : (previous?.staleSince ?? Date.now()) }); - this.capDiscovered(net); + this.capDiscovered(networkID, net); this.notify(networkID); } @@ -263,13 +275,35 @@ export class BootstrapStatusTracker { } } - /** Bound discovered rows per network (drop the oldest) — see MAX_DISCOVERED_PER_NETWORK. */ - private capDiscovered(net: Map): void { + /** + * Bound discovered rows per network — see MAX_DISCOVERED_PER_NETWORK. + * + * Age alone is the wrong order. It looks at neither the row's state nor whether the + * peer is actually in the network, so a flood of freshly invented dead addresses could + * push a live, connected participant out of the list — the stale sweep protects an + * active member deliberately, and this used to undo that. Least useful goes first: + * a non-member whose address failed, then one that has never answered, then a + * non-member that once connected, and members last. + */ + private capDiscovered(networkID: string, net: Map): void { let discovered = 0; for (const p of net.values()) if (p.origin === 'discovered') discovered++; if (discovered <= MAX_DISCOVERED_PER_NETWORK) return; - const oldestFirst = [...net.entries()].filter(([, p]) => p.origin === 'discovered').sort((a, b) => Date.parse(a[1].updatedAt) - Date.parse(b[1].updatedAt)); - for (let i = 0; i < discovered - MAX_DISCOVERED_PER_NETWORK; i++) net.delete(oldestFirst[i]![0]); + const members = this.membersProvider?.(networkID) ?? new Set(); + const rankOf = (p: TrackedPeer): number => { + const pid = p.expectedPeerID ?? p.actualPeerID; + if (pid && members.has(pid)) return 3; + if (p.status === 'connected') return 2; + if (p.status === 'pending') return 1; + return 0; + }; + // Ranked once per row, not inside the comparator — the member lookup would + // otherwise run O(n log n) times over the same snapshot. + const victims = [...net.entries()] + .filter(([, p]) => p.origin === 'discovered') + .map(([key, p]) => ({ key, rank: rankOf(p), age: Date.parse(p.updatedAt) })) + .sort((a, b) => a.rank - b.rank || a.age - b.age); + for (let i = 0; i < discovered - MAX_DISCOVERED_PER_NETWORK; i++) net.delete(victims[i]!.key); } /** Drop a single peer entry directly (used after identity-mismatch purge of discovered peers). */ diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 1cef97b10..53403c25c 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -456,6 +456,8 @@ export class Network { getNode: (): Libp2p | null => this.node, dialByPeerId: (peerID, protocol): Promise => this.dialProtocolByPeerId(peerID, protocol), }); + // Lets the discovered-row cap keep live participants and drop dead addresses first. + this.bootstrapTracker.setMembersProvider((networkID): Set => new Set(this.getTopicPeers(networkID))); this.peerAnnounce = new PeerAnnounceManager({ getNode: (): Libp2p | null => this.node, getPubsub: (): any => this.pubsub, diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 528b9be65..02eb18a65 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -649,3 +649,51 @@ describe('BootstrapStatusTracker — one row per endpoint, whatever the spelling expect(tracker.getStatus(NET)!.peers[0]!.status).toBe('connected'); }); }); + +/** + * The discovered-row cap decides what survives a flood. Dropping simply the oldest row + * let an attacker (or a broken emitter) push a live, connected participant out of the + * list with a burst of freshly invented dead addresses — undoing the protection the + * stale sweep gives an active member. + */ +describe('BootstrapStatusTracker — the cap evicts the least useful row', () => { + const NET = 'net-a'; + const MEMBER = 'PeerMemberAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; + const GHOST = 'PeerGhostBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB'; + + /** Fill the network to exactly the cap with fresh, failed, non-member rows. */ + function floodToCap(tracker: BootstrapStatusTracker, count: number): void { + for (let i = 0; i < count; i++) tracker.recordOutcome(NET, `/ip4/198.51.100.${i % 254}/tcp/${9000 + i}/p2p/${GHOST}`, GHOST, 'timeout', 'no answer', null, 'discovered'); + } + + function survivors(tracker: BootstrapStatusTracker): string[] { + return (tracker.getStatus(NET)?.peers ?? []).map(p => p.multiaddr); + } + + it('drops a fresh dead address before an older connected one', () => { + const tracker = new BootstrapStatusTracker(); + const live = `/ip4/203.0.113.7/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, live, MEMBER, 'connected', null, null, 'discovered'); + // 256 further rows put the network one over the cap. + floodToCap(tracker, 256); + + expect(survivors(tracker)).toContain(live); + }); + + it('keeps an active member even when its row is the oldest and failing', () => { + const tracker = new BootstrapStatusTracker(); + tracker.setMembersProvider(() => new Set([MEMBER])); + const failing = `/ip4/203.0.113.8/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, failing, MEMBER, 'timeout', 'no answer', null, 'discovered'); + floodToCap(tracker, 256); + + expect(survivors(tracker)).toContain(failing); + }); + + it('still enforces the cap', () => { + const tracker = new BootstrapStatusTracker(); + floodToCap(tracker, 300); + + expect(survivors(tracker)).toHaveLength(256); + }); +}); From 0bbf9a3bd3ea6e3838291daa9810c0e8e5f1bf53 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:29:58 +0200 Subject: [PATCH 97/99] fix(network-config): filter self by destination, not by substring --- backend/src/protocol/network-config.ts | 7 ++- .../network-config-self-filter.test.ts | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 backend/tests/unit/protocol/network-config-self-filter.test.ts diff --git a/backend/src/protocol/network-config.ts b/backend/src/protocol/network-config.ts index 84f8c6c08..a6842345d 100644 --- a/backend/src/protocol/network-config.ts +++ b/backend/src/protocol/network-config.ts @@ -25,7 +25,7 @@ import { trace } from '../logger.ts'; import { normalizeTrustedPeerIds, parseAcceptPXThreshold } from './constants.ts'; import { getLocalCidrs, shouldDenyDial, extractFirstIPv4 } from './address-filter.ts'; import { peerIdFromString } from '@libp2p/peer-id'; -import { extractDestinationPeerID } from './multiaddr-utils.ts'; +import { extractDestinationPeerID, destinationPeerIDOf } from './multiaddr-utils.ts'; const { multiaddr: Multiaddr } = await import('@multiformats/multiaddr'); /** A gossipsub direct-peer entry: a peer id and its multiaddrs. */ @@ -75,7 +75,10 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult const bootstrapMultiaddrs: any[] = []; // Unique bootstrap peers computed up-front so gossipsub config below can // pre-populate directPeers from them. - const uniqueBootstrapPeers = [...new Set(bootstrapPeers)].filter(p => !p.includes(myPeerID)); + // Self is decided by the DESTINATION identity, not by "the string mentions us". A + // relayed entry `/…/p2p//p2p-circuit/p2p/` names us as the RELAY hop while + // targeting somebody else, and the substring test threw it away as our own address. + const uniqueBootstrapPeers = [...new Set(bootstrapPeers)].filter(p => destinationPeerIDOf(p) !== myPeerID); const peerExchange = allSettings.network?.peerExchange; const pxEnabled = peerExchange?.enabled === true; const parsedThreshold = parseAcceptPXThreshold(peerExchange?.acceptPXThreshold); diff --git a/backend/tests/unit/protocol/network-config-self-filter.test.ts b/backend/tests/unit/protocol/network-config-self-filter.test.ts new file mode 100644 index 000000000..6983ad2ea --- /dev/null +++ b/backend/tests/unit/protocol/network-config-self-filter.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from 'bun:test'; +import { generateKeyPair } from '@libp2p/crypto/keys'; +import { buildLibp2pConfig } from '../../../src/protocol/network-config.ts'; + +/** + * A bootstrap entry is "ours" only when the address DESTINATION is our own identity. + * A relayed entry `/…/p2p//p2p-circuit/p2p/` mentions us as the relay hop + * while targeting somebody else, and a substring test dropped it as self — silently + * removing a configured peer from the bootstrap list and from gossipsub's direct set. + */ + +const REMOTE = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + +async function build(bootstrapPeers: string[], myPeerID: string) { + const privateKey = await generateKeyPair('Ed25519'); + return buildLibp2pConfig({ + privateKey, + datastore: {}, + allSettings: { network: { mdnsEnabled: false } } as any, + bootstrapPeers, + myPeerID, + }); +} + +describe('buildLibp2pConfig — the self filter reads the destination', () => { + it('keeps a relayed entry that merely passes through our own identity', async () => { + const me = '12D3KooWMyOwnIdentityAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; + const viaUs = `/ip4/198.51.100.4/tcp/9090/p2p/${me}/p2p-circuit/p2p/${REMOTE}`; + + const { bootstrapPeerIDs } = await build([viaUs], me); + + expect([...bootstrapPeerIDs]).toEqual([REMOTE]); + }); + + it('still drops an entry that really targets us', async () => { + const me = REMOTE; + const ours = `/ip4/198.51.100.4/tcp/9090/p2p/${me}`; + + const { bootstrapPeerIDs, bootstrapMultiaddrs } = await build([ours], me); + + expect([...bootstrapPeerIDs]).toEqual([]); + expect(bootstrapMultiaddrs).toEqual([]); + }); +}); From 930758fa323b7fc77e0285a94bd245c6c21e2826 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:32:44 +0200 Subject: [PATCH 98/99] fix(network): single-flight the peer-discovery dial per peer --- backend/src/protocol/network.ts | 25 ++++++++++++++ .../protocol/peer-discovery-handler.test.ts | 34 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 53403c25c..759b98439 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -416,6 +416,16 @@ export class Network { * which a third request saw the address free and duplicated the live dial. */ private inFlightBootstrapDials = new Set(); + /** + * Peer IDs with a `peer:discovery` dial currently outstanding. + * + * Keyed by PEER, not by address, because that is the question discovery asks: mDNS, + * identify and PX all raise an event for the same arrival, each carrying its own + * address list, and the per-peer backoff cannot separate them — it is written only + * after a dial has already failed. Replaced on teardown for the same reason as + * {@link inFlightBootstrapDials}. + */ + private inFlightDiscoveryDials = new Set(); /** * peerID → time we first saw the peer disconnected with ZERO reachable * addresses. Such peers never enter the re-dial path (nothing to dial), so @@ -828,6 +838,18 @@ export class Network { trace(`[NET] discovery dial skipped (quarantined or in backoff): ${peerID.slice(0, 16)}`); return; } + // Single-flight per peer. The backoff above is only written once a dial has + // already FAILED, so several discovery events for one peer — mDNS, identify and + // PX all fire for the same arrival — used to pass it together and start that + // many concurrent dials of the same identity. Captured by reference for the same + // reason the bootstrap claims are: a teardown replaces the set, and releasing + // into the replacement would free the next run's claim. + const inFlight = this.inFlightDiscoveryDials; + if (inFlight.has(peerID)) { + trace(`[NET] discovery dial skipped (already in flight): ${peerID.slice(0, 16)}`); + return; + } + inFlight.add(peerID); const epoch = this.runEpoch; try { @@ -855,6 +877,8 @@ export class Network { // discovery keeps naming is paced like everything else. this.noteRecoveryDialFailure(peerID); trace(`[NET] Failed to dial discovered peer ${peerID.slice(0, 16)}: ${err?.message ?? err}`); + } finally { + inFlight.delete(peerID); } }); @@ -2822,6 +2846,7 @@ export class Network { // must not lock the address out for the next one — nor, by releasing into a shared // Set, steal the claim the next one has taken. A new object does both. this.inFlightBootstrapDials = new Set(); + this.inFlightDiscoveryDials = new Set(); this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); diff --git a/backend/tests/unit/protocol/peer-discovery-handler.test.ts b/backend/tests/unit/protocol/peer-discovery-handler.test.ts index 1cdc7d03b..1ea983f2b 100644 --- a/backend/tests/unit/protocol/peer-discovery-handler.test.ts +++ b/backend/tests/unit/protocol/peer-discovery-handler.test.ts @@ -22,6 +22,7 @@ function bareNetwork(opts: { connected?: boolean; dialFails?: boolean } = {}) { (network as any).runEpoch = 1; (network as any).listeners = []; (network as any).redialSuppressedByNet = new Map(); + (network as any).inFlightDiscoveryDials = new Set(); (network as any).redialBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).dcutrPeers = new Set(); @@ -135,6 +136,7 @@ describe('peer:discovery — a dial that lands after leave-network', () => { (network as any).runEpoch = 1; (network as any).listeners = []; (network as any).redialSuppressedByNet = new Map(); + (network as any).inFlightDiscoveryDials = new Set(); (network as any).redialBackoff = new Map(); (network as any).unreachableQuarantine = new Map(); (network as any).dcutrPeers = new Set(); @@ -234,3 +236,35 @@ describe('peer:discovery — a dial that lands after leave-network', () => { expect(hungUp).toContain(PEER_ID); }); }); + +/** + * mDNS, identify and PX all raise a discovery event for the same arrival. The per-peer + * backoff cannot separate them — it is written only once a dial has already FAILED — so + * without a claim every event started its own dial of the same identity. + */ +describe('peer:discovery — one dial per peer at a time', () => { + it('collapses concurrent events for one peer into a single dial', async () => { + const { network, dialled, fire } = bareNetwork(); + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + (network as any).node.dial = async (multiaddrs: unknown[]): Promise => { + dialled.push(multiaddrs); + await gate; + return {}; + }; + + const first = fire(); + const second = fire(); + const third = fire(); + expect(dialled).toHaveLength(1); + + release(); + await Promise.all([first, second, third]); + + // Claim released — a later event is free to dial again. + await fire(); + expect(dialled).toHaveLength(2); + }); +}); From a03455d332368028be0ad46f127e8cfb95de0e37 Mon Sep 17 00:00:00 2001 From: LuRy Date: Mon, 17 Aug 2026 07:35:31 +0200 Subject: [PATCH 99/99] fix(network): make a configured bootstrap direct as soon as it answers --- backend/src/protocol/network.ts | 47 +++++++++------ .../protocol/network-restart-safety.test.ts | 57 +++++++++++++++++++ 2 files changed, 87 insertions(+), 17 deletions(-) diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index 759b98439..6b1b1ed34 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1585,24 +1585,31 @@ export class Network { // heartbeatInterval). KEEP_ALIVE handles the TCP layer; gossipsub.direct // handles the gossipsub-stream layer. Evicted peers are removed from the // set in purgeStalePeer, so it no longer grows monotonically. - const gossipsub: any = this.pubsub; - if (gossipsub?.direct && typeof gossipsub.direct.add === 'function') { - let added = 0; - for (const peer of allPeers) { - const pid = peer.id.toString(); - if (pid === myID) continue; - if (!connectedIDs.has(pid)) continue; - // A left-network peer that lingers/reappears in the peerStore must not be - // added to the direct set either — its fast reconnect cadence would undo the - // leave-network disconnect (same guard as the bootstrap promotion above). - if (this.isRedialSuppressed(pid)) continue; - if (!gossipsub.direct.has(pid)) { - gossipsub.direct.add(pid); - added++; - } - } - if (added > 0) trace(`[NET] gossipsub direct: added ${added} connected peer(s) to fast-reconnect set`); + let added = 0; + for (const peer of allPeers) { + const pid = peer.id.toString(); + if (pid === myID) continue; + if (!connectedIDs.has(pid)) continue; + if (this.addGossipsubDirectPeer(pid)) added++; } + if (added > 0) trace(`[NET] gossipsub direct: added ${added} connected peer(s) to fast-reconnect set`); + } + + /** + * Put a peer into the gossipsub `direct` set — never PRUNEd, and reconnected on its + * own fast cadence (directConnectTicks × heartbeatInterval). Removed again by + * {@link purgeStalePeer}. Returns whether this call was the one that added it. + * + * A left-network peer that lingers or reappears in the peerStore is refused: the fast + * reconnect cadence would undo the leave-network disconnect. + */ + private addGossipsubDirectPeer(peerID: string): boolean { + const gossipsub: any = this.pubsub; + if (!gossipsub?.direct || typeof gossipsub.direct.add !== 'function') return false; + if (this.isRedialSuppressed(peerID)) return false; + if (gossipsub.direct.has(peerID)) return false; + gossipsub.direct.add(peerID); + return true; } // ========================================================================= @@ -1846,6 +1853,12 @@ export class Network { // wanted — the point at which a discovered ID has earned its place in // the set (see the claim above for why it may not have it yet). if (peerID) this.bootstrapPeerIDs.add(peerID); + // A CONFIGURED bootstrap joins the gossipsub direct set the moment it + // answers. Production starts the node with an empty list — the config-time + // `directPeers` seed never applies — so waiting for the periodic promotion + // left the peer the whole mesh depends on without a fast reconnect, and + // PRUNE-able, for the first ~150 s of every run. + if (peerID && effectiveOrigin === 'configured') this.addGossipsubDirectPeer(peerID); if (pidObj) { await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); } diff --git a/backend/tests/unit/protocol/network-restart-safety.test.ts b/backend/tests/unit/protocol/network-restart-safety.test.ts index 32cd0d2de..5ff33d3e0 100644 --- a/backend/tests/unit/protocol/network-restart-safety.test.ts +++ b/backend/tests/unit/protocol/network-restart-safety.test.ts @@ -210,3 +210,60 @@ describe('Network.addBootstrapPeers — a dial that lands after a restart', () = expect(disconnected).toEqual([PEER_ID]); }); }); + +/** + * Production starts the node with an empty bootstrap list, so the config-time + * `directPeers` seed never applies. A configured bootstrap therefore has to enter the + * gossipsub direct set on acceptance — waiting for the periodic promotion left the peer + * the whole mesh depends on PRUNE-able and without a fast reconnect for ~150 s. + */ +describe('Network.addBootstrapPeers — configured bootstraps become direct peers at once', () => { + function harness(suppressed: string[] = []) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending(): void {}, + recordOutcome(): void {}, + }; + const direct = new Set(); + (network as any).pubsub = { direct }; + (network as any).node = { + peerId: { toString: (): string => 'selfID' }, + getConnections: (): unknown[] => [], + dial: async (): Promise<{ remoteAddr: string }> => ({ remoteAddr: ADDR }), + peerStore: { async merge(): Promise {} }, + }; + (network as any).isPeerNeededByJoinedNetwork = (): boolean => false; + (network as any).isTopicSubscribed = (): boolean => true; + (network as any).rememberBootstrapAddress = (): void => {}; + return { network, direct }; + } + + it('adds a configured bootstrap on the dial that accepts it', async () => { + const { network, direct } = harness(); + + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + + expect([...direct]).toEqual([PEER_ID]); + }); + + it('leaves a merely discovered peer to the periodic promotion', async () => { + const { network, direct } = harness(); + + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + + expect([...direct]).toEqual([]); + }); +});