diff --git a/backend/src/api/factory-reset-orchestrator.ts b/backend/src/api/factory-reset-orchestrator.ts index be773f34e..20ab81b2f 100644 --- a/backend/src/api/factory-reset-orchestrator.ts +++ b/backend/src/api/factory-reset-orchestrator.ts @@ -68,10 +68,10 @@ export function buildFactoryResetHandler(deps: FactoryResetOrchestratorDeps): (p for (const id of enabledDownloads) triggerEnableDownload(id); }; - // Each selected category is wiped INDEPENDENTLY (a failure in one never blocks - // the others) and the node is always brought back up best-effort, so a partial - // failure can't leave networks stopped. Per-category outcomes go to the FE, - // which surfaces one notification per category. See runFactoryReset. + // `prepare` is a barrier: if the transfers or the node cannot be stopped, every + // wipe that works on state the node owns is skipped and nothing is restarted. + // Categories that do run are independent of each other. Per-category and + // per-phase outcomes go to the FE, one notification each. See runFactoryReset. const response = await runFactoryReset({ prepare: async () => { if (wipeDownloads || restartNode) { diff --git a/backend/src/api/factory-reset.ts b/backend/src/api/factory-reset.ts index 77426d9f7..fa2f8e741 100644 --- a/backend/src/api/factory-reset.ts +++ b/backend/src/api/factory-reset.ts @@ -1,15 +1,18 @@ -import { type FactoryResetCategory, type FactoryResetResult, type FactoryResetResponse } from '@shared'; +import { type FactoryResetCategory, type FactoryResetResult, type FactoryResetPhaseResult, type FactoryResetResponse } from '@shared'; /** - * Operations a factory reset performs. `prepare`/`restart` are best-effort - * infrastructure run around the wipes (logged but never reported as categories); - * the five category functions are each run INDEPENDENTLY. Omit a function to skip - * that step (e.g. an unselected category). + * Operations a factory reset performs. `prepare` and `restart` are not categories but + * the infrastructure around the wipes; the five category functions are each run + * independently of one another. Omit a function to skip that step (e.g. an unselected + * category). */ export interface FactoryResetOps { - /** Stop transfers / networks before the wipes. Best-effort. */ + /** + * Stop transfers / networks before the wipes. A PREREQUISITE, not a best-effort + * extra: everything in {@link REQUIRES_PREPARE} is only safe once it has succeeded. + */ prepare?: (() => Promise | void) | undefined; - /** Bring the node + surviving transfers back after the wipes. Best-effort. */ + /** Bring the node + surviving transfers back after the wipes. Skipped if `prepare` failed. */ restart?: (() => Promise | void) | undefined; settings?: (() => Promise | void) | undefined; identity?: (() => Promise | void) | undefined; @@ -23,30 +26,72 @@ export interface FactoryResetOps { // the per-category notifications predictable on the FE. const CATEGORY_ORDER: FactoryResetCategory[] = ['downloads', 'networks', 'peers', 'identity', 'settings']; +/** + * Categories that may only run once `prepare` has actually stopped the transfers and the + * node. Four of the five wipe state the running node owns — its datastore, its peerstore, + * its identity, the files live transfers are writing — so running them over a node that + * could not be stopped is the corruption this barrier exists to prevent. `settings` is the + * only wipe that touches neither the node nor the transfers. + */ +const REQUIRES_PREPARE: ReadonlySet = new Set(['downloads', 'networks', 'peers', 'identity']); + +const PREPARE_FAILED_SKIP = 'skipped: transfers and the node could not be stopped safely'; +const RESTART_SKIPPED = 'skipped: nothing was stopped, so nothing may be started'; +const PREPARE_MISSING = 'no prepare step was supplied, so nothing was stopped'; + function errMsg(e: unknown): string { return e instanceof Error ? e.message : String(e); } /** - * Run a factory reset where every selected category is wiped INDEPENDENTLY: a - * failure in one category is caught, recorded, and never stops the remaining - * categories. `prepare` runs first and `restart` last — both best-effort, so their - * failure is logged but does not abort the wipes (the node is always brought back - * up). Returns one {@link FactoryResetResult} per selected category plus an overall - * `success` flag (true iff every selected category passed). + * Run a factory reset. + * + * `prepare` is a hard barrier. It stops the transfers and the libp2p node, and every + * destructive category listed in {@link REQUIRES_PREPARE} depends on that having actually + * happened — a wipe of the datastore, the peerstore or the download tables underneath a + * node still holding them corrupts both. So a failed `prepare` skips those categories, + * skips the restart (there is nothing safe to restart onto) and forces `success: false`. + * It used to be merely logged, which let the reset proceed to wipe and then bring up a + * second node over whatever the first one still owned. + * + * A caller that supplies no `prepare` at all gets the same treatment, because it has proved + * exactly as little: `prepared` starts false, and selecting any {@link REQUIRES_PREPARE} + * category without a barrier records a failed prepare phase and skips that category. The + * absence of a barrier used to count as a passed one, so this function would happily wipe + * downloads, networks, peers and the identity out from under a running node and report + * `success: true`. `prepare` stays optional only so a settings-only reset — which touches + * neither the node nor the transfers — need not stop anything. + * + * Categories that DO run are independent of each other: a failure in one is recorded and + * never stops the rest. Returns one {@link FactoryResetResult} per selected category, one + * {@link FactoryResetPhaseResult} per phase, and an overall `success` that is true only + * when every one of them passed. */ export async function runFactoryReset(ops: FactoryResetOps): Promise { + const phases: FactoryResetPhaseResult[] = []; + let prepared = false; if (ops.prepare) { try { await ops.prepare(); + prepared = true; + phases.push({ phase: 'prepare', ok: true }); } catch (e) { - console.error(`[factoryReset] prepare failed (continuing): ${errMsg(e)}`); + const detail = errMsg(e); + console.error(`[factoryReset] prepare failed — destructive categories skipped: ${detail}`); + phases.push({ phase: 'prepare', ok: false, detail }); } + } else if (CATEGORY_ORDER.some(category => ops[category] && REQUIRES_PREPARE.has(category))) { + console.error(`[factoryReset] ${PREPARE_MISSING} — destructive categories skipped`); + phases.push({ phase: 'prepare', ok: false, detail: PREPARE_MISSING }); } const results: FactoryResetResult[] = []; for (const category of CATEGORY_ORDER) { const fn = ops[category]; if (!fn) continue; // category not selected + if (!prepared && REQUIRES_PREPARE.has(category)) { + results.push({ category, ok: false, detail: PREPARE_FAILED_SKIP }); + continue; + } try { await fn(); results.push({ category, ok: true }); @@ -57,11 +102,21 @@ export async function runFactoryReset(ops: FactoryResetOps): Promise r.ok), results }; + return { success: results.every(r => r.ok) && phases.every(p => p.ok), results, phases }; } diff --git a/backend/src/api/lishnets.ts b/backend/src/api/lishnets.ts index eb7113d01..b8e0df8c7 100644 --- a/backend/src/api/lishnets.ts +++ b/backend/src/api/lishnets.ts @@ -10,11 +10,11 @@ interface LISHnetsHandlers { get: (p: { networkID: string }) => LISHNetworkConfig | undefined; exists: (p: { networkID: string }) => boolean; add: (p: { network: LISHNetworkConfig }) => Promise; - update: (p: { network: LISHNetworkConfig }) => boolean; + update: (p: { network: LISHNetworkConfig }) => Promise; delete: (p: { networkID: string }) => Promise; addIfNotExists: (p: { network: LISHNetworkDefinition }) => Promise; - import: (p: { networks: LISHNetworkDefinition[] }) => number; - replace: (p: { networks: LISHNetworkConfig[] }) => boolean; + import: (p: { networks: LISHNetworkDefinition[] }) => Promise; + replace: (p: { networks: LISHNetworkConfig[] }) => Promise; exportToFile: (p: { networkID: string; filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }) => Promise; exportAllToFile: (p: { filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }) => Promise; importFromFile: (p: { path: string; enabled?: boolean }) => Promise; @@ -54,7 +54,7 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, assert(p, ['network']); return networks.add(p.network); } - function update(p: { network: LISHNetworkConfig }): boolean { + async function update(p: { network: LISHNetworkConfig }): Promise { assert(p, ['network']); return networks.update(p.network); } @@ -66,13 +66,13 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, assert(p, ['network']); return networks.addIfNotExists(p.network); } - function importNetworks(p: { networks: LISHNetworkDefinition[] }): number { + async function importNetworks(p: { networks: LISHNetworkDefinition[] }): Promise { assert(p, ['networks']); return networks.importNetworks(p.networks); } - function replace(p: { networks: LISHNetworkConfig[] }): boolean { + async function replace(p: { networks: LISHNetworkConfig[] }): Promise { assert(p, ['networks']); - networks.replace(p.networks); + await networks.replace(p.networks); return true; } async function exportToFile(p: { networkID: string; filePath: string; minifyJSON?: boolean; compress?: boolean; compressionAlgorithm?: CompressionAlgorithm }): Promise { @@ -116,13 +116,19 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, } async function setEnabled(p: { networkID: string; enabled: boolean }): Promise { assert(p, ['networkID', 'enabled']); - const net = networks.get(p.networkID); - const success = await networks.setEnabled(p.networkID, p.enabled); - if (success && net) { - const event = p.enabled ? 'lishnets:joined' : 'lishnets:left'; - broadcast(event, { networkID: p.networkID, name: net.name }); - } - return { success }; + const result = await networks.setEnabled(p.networkID, p.enabled); + // Only a settled transition is broadcast, and the event names the state the network + // actually ended in. Broadcasting on "the network exists" plus the REQUESTED flag + // announced a join for a request a newer one had already overruled, and repeated the + // event for an enable of an already-enabled network. + // + // The name comes from the operation, not from a get() of our own. Reading the row + // before the await missed a network that was still being added — undefined, so the + // join it really did perform was never broadcast — and carried the pre-rename name + // when an edit was queued ahead of the enable. Reading it after the await races the + // next write instead. + if (result.transitioned && result.network) broadcast(result.joined ? 'lishnets:joined' : 'lishnets:left', result.network); + return { success: result.found }; } async function connect(p: { multiaddr: string }): Promise { assert(p, ['multiaddr']); diff --git a/backend/src/db/lishnets.ts b/backend/src/db/lishnets.ts index 82124a135..030a69249 100644 --- a/backend/src/db/lishnets.ts +++ b/backend/src/db/lishnets.ts @@ -1,5 +1,36 @@ import { type Database } from 'bun:sqlite'; import { type LISHNetworkConfig, type LISHNetworkDefinition } from '@shared'; +import { canonicalMultiaddr } from '../protocol/multiaddr-utils.ts'; + +/** + * Normalise a bootstrap list: drop non-strings and blanks, trim, and keep one entry per + * canonical address. + * + * Trimming matters because the list usually came from a text field — a value with stray + * whitespace passed the old blank check and then failed to parse at dial time. + * Deduplicating matters because two spellings of one address (DNS case, trailing dot, + * expanded vs compressed IPv6) would otherwise each get their own forced probe and their + * own status row for the same endpoint. + * + * It lives HERE, at the write boundary, and is applied by every writer below. As a helper + * the callers were free to skip, only the two edit paths used it, so import, add and + * wholesale replace could all store values that the rest of the system then had to cope + * with. + */ +export function cleanBootstrapList(peers: string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const raw of peers) { + if (typeof raw !== 'string') continue; + const trimmed = raw.trim(); + if (trimmed.length === 0) continue; + const key = canonicalMultiaddr(trimmed); + if (seen.has(key)) continue; + seen.add(key); + out.push(trimmed); + } + return out; +} export function initLISHnetsTables(db: Database): void { db.run(` @@ -32,6 +63,11 @@ function getInternalID(db: Database, networkID: string): number | null { return row?.id ?? null; } +/** The one place a bootstrap list is written — so the one place it is normalised. */ +function writeBootstrapPeers(db: Database, internalID: number, peers: string[]): void { + for (const peer of cleanBootstrapList(peers)) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); +} + function getBootstrapPeers(db: Database, internalID: number): string[] { return db .query<{ address: string }, [number]>('SELECT address FROM lishnets_peers WHERE id_lishnets = ? ORDER BY id') @@ -95,7 +131,7 @@ export function addLISHnet(db: Database, network: LISHNetworkConfig): boolean { ); const internalID = Number(result.lastInsertRowid); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); }); tx(); return true; @@ -114,7 +150,7 @@ export function updateLISHnet(db: Database, network: LISHNetworkConfig): boolean // Replace peers db.run('DELETE FROM lishnets_peers WHERE id_lishnets = ?', [internalID]); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); }); tx(); return true; @@ -163,7 +199,7 @@ export function replaceLISHnets(db: Database, networks: LISHNetworkConfig[]): vo [network.networkID, network.name, network.description || null, network.enabled ? 1 : 0, network.created || null] ); const internalID = Number(result.lastInsertRowid); - for (const peer of network.bootstrapPeers) db.run('INSERT INTO lishnets_peers (id_lishnets, address) VALUES (?, ?)', [internalID, peer]); + writeBootstrapPeers(db, internalID, network.bootstrapPeers); } }); tx(); diff --git a/backend/src/lishnet/lishnets.ts b/backend/src/lishnet/lishnets.ts index ab42cc684..031c70d9e 100644 --- a/backend/src/lishnet/lishnets.ts +++ b/backend/src/lishnet/lishnets.ts @@ -1,10 +1,36 @@ import { type Database } from 'bun:sqlite'; -import { Network } from '../protocol/network.ts'; +import { Mutex } from 'async-mutex'; +import { Network, normalizeMultiaddrForCompare } from '../protocol/network.ts'; import { Utils } from '../utils.ts'; import { type DataServer } from '../lish/data-server.ts'; import { type Settings } from '../settings.ts'; import { type ILISHNetwork, type LISHNetworkConfig, type LISHNetworkDefinition, type PeerConnectionInfo, type IMeshHealth, type BootstrapStatus, CodedError, ErrorCodes } from '@shared'; -import { lishnetExists, getLISHnet, listLISHnets, listEnabledLISHnets, addLISHnet, updateLISHnet, deleteLISHnet, setLISHnetEnabled, addLISHnetIfNotExists, importLISHnets, upsertLISHnet, replaceLISHnets } from '../db/lishnets.ts'; +import { cleanBootstrapList, lishnetExists, getLISHnet, listLISHnets, listEnabledLISHnets, addLISHnet, updateLISHnet, deleteLISHnet, setLISHnetEnabled, addLISHnetIfNotExists, importLISHnets, upsertLISHnet, replaceLISHnets } from '../db/lishnets.ts'; + +/** + * Outcome of {@link Networks.setEnabled}. + * + * A bare boolean could not say what happened. It meant "the network exists", and callers + * read it as "your change was applied" — so a request superseded by a newer one, and an + * idempotent one that changed nothing, both looked like a settled transition and made the + * API broadcast a join or leave that had not occurred. + */ +export interface SetEnabledResult { + /** Whether the lishnet exists at all. */ + found: boolean; + /** Whether this call actually settled a change of join state. */ + transitioned: boolean; + /** The join state the lishnet is in now, whoever settled it. */ + joined: boolean; + /** + * Identity of the row as it stood inside the critical section, for the event the API + * broadcasts. The handler used to read the row itself before awaiting this call, which + * raced the catalog in both directions: a network still being added read as undefined + * and its join was never broadcast, and a rename queued ahead of the enable made the + * event carry the name the network no longer has. + */ + network?: { networkID: string; name: string }; +} /** * Manages lishnets (logical network groups) on top of a single shared Network (libp2p) node. @@ -17,6 +43,49 @@ export class Networks { // Track which lishnets are currently joined (subscribed) private joinedNetworks: Set = new Set(); + /** + * One lock per lishnet, so join and leave of the SAME network never interleave. + * + * Both of them await for a long time — a join waits on bootstrap dials, a leave + * disconnects peers one at a time — while mutating `joinedNetworks`, the pubsub + * subscription and the redial suppression that the other one reads. Overlapping + * runs left those four disagreeing with the database and with each other. + */ + private readonly networkOperations = new Map(); + /** + * Taken by EVERY public write, before any per-ID lock. + * + * Two things need it. A multi-network write has to see a stable set of IDs while it + * decides what it is rewriting, so without this a network created between the snapshot + * and the write was rewritten out of existence behind the back of the add that was still + * joining it. And every writer must reach its database write in the order the requests + * arrived: `Mutex` dispatches its waiters first come, first served, and each public + * writer enqueues here before it awaits anything else, so one gate for everyone makes + * arrival order and write order the same. + * + * Held for the DATABASE phase only — see {@link inCatalog} and {@link reconcile}. It is + * never held while a per-ID lock is taken, so no cycle is possible. + */ + private readonly catalogMutex = new Mutex(); + /** + * The join/leave state last announced to higher layers, per lishnet. + * + * Two consecutive writes can both converge the same lishnet, and the second usually + * finds the runtime already where it wants it. Announcing the OUTCOME rather than the + * operation covers that: exactly one event per actual change, none for a write that + * changed nothing. Unset reads as "not joined", and the startup join seeds it directly — + * startup itself stays silent (it has its own resume path) while a later disable still + * has a `true` to change away from. + */ + private readonly announcedJoined = new Map(); + + /** + * True from the synchronous start of {@link stopAllNetworks} until the next + * {@link startEnabledNetworks}. Startup joins consult it because the node's own + * `isRunning()` cannot answer for the window before `stop()` has taken its mutex. + */ + private shuttingDown = false; + // Callback for peer count changes private _onPeerCountChange: ((counts: { networkID: string; count: number }[]) => void) | null = null; // Callback for bootstrap status changes @@ -93,7 +162,7 @@ export class Networks { * The node always starts, even if no lishnets are enabled. */ async startEnabledNetworks(): Promise { - const enabled = this.getEnabled(); + this.shuttingDown = false; // Start the node with no preset bootstrap list — bootstrap dials happen // per-network below via addBootstrapPeers so per-network status tracking @@ -101,37 +170,206 @@ export class Networks { // (Previous behaviour used a flat preset list that bypassed our tracking.) await this.network.start([]); - // Subscribe to topics for all enabled lishnets and dial their bootstrap peers - // with networkID context so bootstrap status counters get populated. - for (const net of enabled) { - this.network.subscribeTopic(net.networkID); - this.joinedNetworks.add(net.networkID); - if (net.bootstrapPeers.length > 0) { - // Fire-and-forget so a slow / unreachable network does not delay startup of the others. - this.network.addBootstrapPeers(net.bootstrapPeers, net.networkID, 'configured').catch(err => { - console.error(`[Networks] addBootstrapPeers for ${net.networkID} failed:`, err?.message ?? err); + // The enabled list is read AFTER the start, not before it. Reading it first meant + // startup worked from a snapshot taken before a long await: an API disable or + // delete arriving during the start reconciled against a runtime that had joined + // nothing yet — so it had nothing to leave — and then this loop subscribed the + // network anyway, from a copy of a row that no longer said what it used to. + // Under the catalog mutex for the whole loop, so no API write and no shutdown can + // interleave with the networks coming up — see {@link catalogMutex}. + await this.catalogMutex.runExclusive(async () => { + for (const net of this.getEnabled()) { + await this.operationLock(net.networkID).runExclusive(() => { + // Re-read under the lock as well: an earlier network's turn is another await. + const row = this.get(net.networkID); + if (row?.enabled !== true) return; + // A stop between the start above and this subscribe leaves the call a no-op on + // a dead node, but `joinedNetworks` would still claim membership — the wrapper + // reporting a joined network whose node is not running and whose topic is not + // subscribed. + if (!this.canJoin()) return; + if (!this.network.subscribeTopic(row.networkID)) return; + this.joinedNetworks.add(row.networkID); + // Startup itself announces nothing, but a later disable has to have a joined + // state to change away from — otherwise its leave looks like a no-op. + this.announcedJoined.set(row.networkID, true); + if (row.bootstrapPeers.length > 0) { + // Fire-and-forget so a slow / unreachable network does not delay startup of the others. + this.network.addBootstrapPeers(row.bootstrapPeers, row.networkID, 'configured').catch(err => { + console.error(`[Networks] addBootstrapPeers for ${row.networkID} failed:`, err?.message ?? err); + }); + } + console.log(`✓ Joined lishnet: ${row.name} (${row.networkID})`); }); } - console.log(`✓ Joined lishnet: ${net.name} (${net.networkID})`); - } + }); + } + + /** + * Whether a runtime join may go ahead right now. + * + * Both halves matter. `shuttingDown` is set synchronously by {@link stopAllNetworks}, + * so it covers the window before `Network.stop()` has even reached its own mutex, in + * which `isRunning()` still answers true; `isRunning()` covers a node that is down for + * any other reason, including a stop whose teardown failed. + */ + private canJoin(): boolean { + return !this.shuttingDown && this.network.isRunning(); } /** * Enable/disable a lishnet. Starts the node if needed, subscribes/unsubscribes topics. */ - async setEnabled(id: string, enabled: boolean): Promise { - if (!lishnetExists(this.db, id)) return false; + async setEnabled(id: string, enabled: boolean): Promise { + const staged = await this.inCatalog(() => { + if (!lishnetExists(this.db, id)) return undefined; + const previous = this.get(id); + setLISHnetEnabled(this.db, id, enabled); + // Named from the row this write landed on, not from a read the caller took outside + // the lock — see {@link SetEnabledResult.network}. + return { previous, row: this.get(id) }; + }); + if (!staged) return { found: false, transitioned: false, joined: false }; + const transitioned = await this.reconcile(id, staged.previous); + const result: SetEnabledResult = { found: true, transitioned, joined: this.joinedNetworks.has(id) }; + if (staged.row) result.network = { networkID: staged.row.networkID, name: staged.row.name }; + return result; + } + + /** + * Phase one of every lishnet write: the DATABASE, under {@link catalogMutex} alone. + * + * `body` is synchronous and must not touch the network. The catalog used to be held for + * the runtime phase as well, and that phase is slow — a join awaits a sequential dial of + * every bootstrap address, seconds each, and a leave one hangUp per peer. Editing an + * unrelated lishnet, an add, a delete, an import, another replace, a shutdown and a + * factory reset all queued behind whichever single network happened to be dialing, + * which presented as a frozen shutdown. + */ + private async inCatalog(body: () => T): Promise { + return await this.catalogMutex.runExclusive(async () => body()); + } - setLISHnetEnabled(this.db, id, enabled); + /** + * Phase two: converge one lishnet's runtime on the row phase one left behind, under that + * lishnet's own lock and nothing else. + * + * Safe outside the catalog precisely because {@link reconcileLocked} takes its desired + * state from the database rather than from a value its caller captured: whichever + * reconcile holds the lock last converges on the row that was written last, whatever + * order the two finished their network work in. + */ + private async reconcile(id: string, previous: LISHNetworkConfig | undefined): Promise { + const transitioned = await this.operationLock(id).runExclusive(() => this.reconcileLocked(id, previous)); + this.forgetIfGone(id); + return transitioned; + } + + /** + * Drop the per-lishnet bookkeeping of a lishnet that no longer exists. + * + * {@link networkOperations} and {@link announcedJoined} are keyed by arbitrary network + * IDs and nothing used to remove an entry, so creating and deleting networks over a long + * uptime grew both without bound. + * + * Called with the lock already RELEASED, and only when nothing holds or waits on it — + * `isLocked()` covers both. Deleting a mutex somebody is queued on would hand the next + * caller a second, independent mutex for the same lishnet, which is worse than a leak. + */ + private forgetIfGone(id: string): void { + // A lishnet still joined has runtime state to describe, however little the database + // has left to say about it. + if (this.get(id) !== undefined || this.joinedNetworks.has(id)) return; + this.announcedJoined.delete(id); + const lock = this.networkOperations.get(id); + if (lock && !lock.isLocked()) this.networkOperations.delete(id); + } + + /** The lock guarding one lishnet's whole transition — see {@link networkOperations}. */ + private operationLock(id: string): Mutex { + let lock = this.networkOperations.get(id); + if (!lock) { + lock = new Mutex(); + this.networkOperations.set(id, lock); + } + return lock; + } - if (enabled) await this.joinNetwork(id); - else await this.leaveNetwork(id); + /** + * Bring the runtime in line with what the DATABASE now says about one lishnet. + * + * Every writer used to be responsible for this itself, and most of them simply were + * not: importing an already-joined network rewrote its bootstrap list in the database + * while the node went on dialing the old one, and importing an active network as + * disabled left it joined until the next restart. + * + * The stored row is the desired state and this converges on it. It is deliberately NOT + * handed the value its caller asked for: two writes over one lishnet serialise on its + * lock, and each converging on the row as it stands when its turn comes is what makes + * the last write the one that decides, whatever order the runtime work finishes in. + * + * An operation that starts also always finishes. It used to abandon itself as soon as a + * newer request had merely ARRIVED — which is not the same as a newer request having + * taken the work over. Two identical disables were enough: the first stopped half-way + * through its peer cleanup, and the second found the network already unsubscribed and + * returned at once, leaving the keep-alive tags, peerStore records and connections of a + * network nobody is in installed with nobody left to remove them. Finishing and then + * applying the next request costs one redundant pass over a rare user action. + * + * `previous` is the row as it was before the write. It says whether the bootstrap list + * moved, and a leave needs it because the cleanup has to run over the list the network + * was joined WITH, which the new row no longer holds — and may not exist at all. + * + * Callers hold the lishnet's operation lock. Returns whether this call settled an actual + * change of join state. + */ + private async reconcileLocked(id: string, previous: LISHNetworkConfig | undefined): Promise { + const next = this.get(id); + const wantJoined = next?.enabled === true; + const joined = this.joinedNetworks.has(id); + const before = Networks.cleanBootstrapList(previous?.bootstrapPeers ?? []); + const after = Networks.cleanBootstrapList(next?.bootstrapPeers ?? []); + // A list change is picked up first so a network that is about to be joined is + // joined against the pruned state — except when we are on our way OUT of it, where + // the leave resets the whole status anyway and a dial would be pure waste. + if (!(joined && !wantJoined) && before.join('\n') !== after.join('\n')) this.syncBootstrapRuntime(id, previous?.bootstrapPeers ?? [], after); + if (joined !== wantJoined) { + if (wantJoined) await this.joinNetwork(id); + else await this.leaveNetwork(id, previous ? before : undefined); + } + return this.announce(id, this.joinedNetworks.has(id)); + } + /** + * Tell higher layers about a settled join/leave, once per actual change. Returns whether + * this call was that change — the single source of truth for "something transitioned", + * which the API needs before it broadcasts a join or leave of its own. + * + * The observers run synchronously here, still under the lishnet's operation lock, so they + * stay in the order the transitions happened. What they may NOT do is change the outcome: + * the transfer-layer callbacks iterate downloaders and mutate them, and a throw used to + * come back out of the whole operation as a failed RPC — after the database and the + * runtime had already moved and `announcedJoined` already held the new state. Retrying + * the request then found nothing left to announce, so the observer was never re-run and + * the event never reached the client, for a network that really had joined or left. + */ + private announce(id: string, joined: boolean): boolean { + if ((this.announcedJoined.get(id) ?? false) === joined) return false; + this.announcedJoined.set(id, joined); + try { + if (joined) this._onNetworkJoined?.(id); + else this._onNetworkLeft?.(id); + } catch (err: any) { + console.error(`[Networks] ${joined ? 'onNetworkJoined' : 'onNetworkLeft'} observer for ${id} threw:`, err?.message ?? err); + } return true; } /** * Join a lishnet (subscribe to its topic, add bootstrap peers). + * + * Announcing the join is {@link reconcileLocked}'s job, not this one's — see + * {@link announcedJoined} for why the outcome is what gets announced. */ private async joinNetwork(id: string): Promise { if (this.joinedNetworks.has(id)) { @@ -139,13 +377,25 @@ export class Networks { return; } + // A join queued behind a slow operation can reach this point after the node has been + // told to stop. subscribeTopic is then a logged no-op, and recording the ID anyway + // left `joinedNetworks` claiming a membership with no subscription behind it — which + // the next startup reads as "already joined" and skips. + if (!this.canJoin()) { + console.log(`Not joining lishnet ${id}: the node is not running`); + return; + } + // Subscribe to the topic first (register interest), then dial bootstrap peers. // Note: the StreamStateError crash from gossipsub is caused by an internal // race condition when peers connect and disconnect rapidly (flapping). // Gossipsub reacts to peer:connect events and tries to send subscriptions // on a stream that may already be closing. This cannot be fixed by call // ordering — the process-level error handlers in app.ts are the safety net. - this.network.subscribeTopic(id); + if (!this.network.subscribeTopic(id)) { + console.log(`Not joining lishnet ${id}: the topic subscription was refused`); + return; + } this.joinedNetworks.add(id); // Rejoin is an explicit "I want peers back" — lift the redial suppression for // THIS lishnet's left peers (bootstrap and content) so maintenance and discovery @@ -155,11 +405,16 @@ export class Networks { const net = this.get(id); if (net && net.bootstrapPeers.length > 0) await this.network.addBootstrapPeers(net.bootstrapPeers, id, 'configured'); - console.log(`✓ Joined lishnet: ${net?.name ?? id}`); + // The dials above take seconds. A node that went down during them owns neither the + // subscription nor the connections this join was building, so the membership claim + // has to go with it rather than survive into the next run. + if (!this.canJoin()) { + this.joinedNetworks.delete(id); + console.log(`Abandoning join of lishnet ${id}: the node went down during its bootstrap dials`); + return; + } - // Notify higher layers (e.g. transfer) so downloads suspended when this - // lishnet was last left can resume now that it is joined again. - this._onNetworkJoined?.(id); + console.log(`✓ Joined lishnet: ${net?.name ?? id}`); } /** @@ -179,12 +434,17 @@ export class Networks { return ids; } - /** Configured-bootstrap peer IDs of a single network. */ - private configuredBootstrapPeerIDsOf(networkID: string): Set { - return new Set(Networks.bootstrapPeerIDsOf(this.get(networkID)?.bootstrapPeers ?? [])); + /** Configured-bootstrap peer IDs of every joined network except `exceptID`. */ + /** Canonical bootstrap ADDRESSES configured for every joined network except `exceptID`. */ + private configuredBootstrapAddressesElsewhere(exceptID: string): Set { + const out = new Set(); + for (const nid of this.joinedNetworks) { + if (nid === exceptID) continue; + for (const address of Networks.cleanBootstrapList(this.get(nid)?.bootstrapPeers ?? [])) out.add(normalizeMultiaddrForCompare(address)); + } + return out; } - /** Configured-bootstrap peer IDs of every joined network except `exceptID`. */ private configuredBootstrapPeerIDsElsewhere(exceptID: string): Set { const out = new Set(); for (const nid of this.joinedNetworks) { @@ -194,9 +454,23 @@ export class Networks { return out; } - private async leaveNetwork(id: string): Promise { + /** + * Leave a lishnet: unsubscribe its topic and undo everything the membership installed. + * + * Runs to completion once it has started. The loops below used to check whether a newer + * request had arrived and return if so, which left the cleanup half-done for a successor + * that then had nothing to do — see {@link reconcileLocked}. + */ + private async leaveNetwork(id: string, outgoingBootstrap?: string[]): Promise { if (!this.joinedNetworks.has(id)) return; + // The list we are leaving, NOT whatever the database holds now. Every caller on the + // disable path writes the row before the runtime catches up — an edit that swaps the + // bootstraps and disables in one go, or a `replace()`/`delete()` that removes the row + // outright — so re-reading here cleaned up the INCOMING list (or nothing at all) and + // left the outgoing addresses installed: still exempt from eviction, still redialled. + const outgoing = Networks.cleanBootstrapList(outgoingBootstrap ?? this.get(id)?.bootstrapPeers ?? []); + // Snapshot the topic subscribers BEFORE unsubscribing — unsubscribeTopic // tears the topic out of pubsub, after which getTopicPeers(id) returns []. // Union with recently-seen members (TTL) so a content peer that is momentarily @@ -208,6 +482,14 @@ export class Networks { this.network.unsubscribeTopic(id); this.joinedNetworks.delete(id); + // Abandon any bootstrap job still walking this network's list — left half-way + // through, it would keep dialing peers of a network we just left and clear the + // redial suppression the loop below is about to apply — and drop the status rows + // with it. Those rows describe a membership that has ended: keeping them meant a + // later rejoin opened on the previous session's connected/error/discovered results + // until fresh dials happened to overwrite each one. Reset publishes an empty list, + // which is what the UI should show for a network this node is not in. + this.network.resetBootstrapStatus(id); // Subscribers of any OTHER joined lishnet must stay connected (shared // infrastructure). Compute this set BEFORE the bootstrap cleanup so that loop @@ -234,8 +516,17 @@ export class Networks { // if it still subscribes another joined lishnet, or if it is an active circuit // relay we depend on. disconnectPeer is a safe no-op hangUp for an unconnected // peer and always strips keep-alive + suppresses redial. + // Address-level cleanup first, because the identity-level loop below cannot do it. + // One peer can be configured in two networks under two DIFFERENT addresses; on + // leaving the first, `stillConfigured` says the identity is in use elsewhere and + // skips its cleanup entirely — so the left network's own address went on counting + // as a configured bootstrap: force-dialed by the parked probe, exempt from the + // stale sweep, and disagreeing with what the UI shows as configured. + const configuredElsewhere = this.configuredBootstrapAddressesElsewhere(id); + this.network.pruneBootstrapAddresses(outgoing.filter(address => !configuredElsewhere.has(normalizeMultiaddrForCompare(address)))); + const stillConfigured = this.configuredBootstrapPeerIDsElsewhere(id); - for (const pid of this.configuredBootstrapPeerIDsOf(id)) { + for (const pid of new Set(Networks.bootstrapPeerIDsOf(outgoing))) { if (stillConfigured.has(pid)) continue; this.network.pruneConfiguredBootstrapPeer(pid); if (stillJoinedPeers.has(pid)) continue; @@ -258,19 +549,41 @@ export class Networks { const net = this.get(id); console.log(`✓ Left lishnet: ${net?.name ?? id}`); - - // Notify higher layers (e.g. transfer) so downloads bound exclusively to - // this lishnet can be stopped. - this._onNetworkLeft?.(id); } /** * Stop all networks and the shared node. */ async stopAllNetworks(): Promise { - this.joinedNetworks.clear(); - await this.network.stop(); - console.log('✓ All lishnets left and node stopped'); + // Set before anything is awaited — `Network.stop()` does not reach its own mutex + // until the next microtask, so `isRunning()` alone still reads true for a moment + // and a startup loop in that moment would subscribe onto a node about to die. + this.shuttingDown = true; + // The catalog mutex covers the DATABASE phase of every writer, not their network + // work, so it is `shuttingDown` — set above, before anything is awaited — that keeps + // a join out of the way: {@link canJoin} is consulted before the subscribe and again + // after the bootstrap dials, so a join running concurrently with this either never + // subscribes or drops the membership claim it was building. What the mutex adds is + // that no new row can be written, and no reconcile started, from the moment the node + // begins to go down. + await this.catalogMutex.runExclusive(async () => { + // Cleared only once the node is provably down. Discarding the membership first + // meant a stop that failed — leaving the node alive and the wrapper `failed` — + // still left this layer claiming it was in no lishnet and had announced nothing. + // `leaveNetwork()` begins with "not joined, nothing to do", so disabling one of + // those networks afterwards wrote `enabled=false` and then unsubscribed nothing, + // disconnected nobody and dropped no keep-alive tag, while the node went on + // subscribed to the topic. + await this.network.stop(); + // Per-run, like `joinedNetworks` itself. Surviving a stop left the map claiming + // networks were still announced as joined, so after a restart a network that came + // back disabled never produced the "left" event its subscribers were waiting for, + // and a rejoin of one that had been announced before the stop produced no event + // either — the runtime had changed and nobody was told. + this.joinedNetworks.clear(); + this.announcedJoined.clear(); + console.log('✓ All lishnets left and node stopped'); + }); } /** @@ -330,7 +643,10 @@ export class Networks { networkID: data.networkID, name: data.name, description: data.description || '', - bootstrapPeers: Array.isArray(data.bootstrapPeers) ? data.bootstrapPeers.filter(p => typeof p === 'string' && p.trim()) : [], + // Cleaned here as well as on write: this shape is also handed straight back to the + // caller as an import preview, and a preview that still shows the untrimmed value + // describes something other than what would be stored. + bootstrapPeers: Array.isArray(data.bootstrapPeers) ? cleanBootstrapList(data.bootstrapPeers) : [], created: data.created || new Date().toISOString(), }; } @@ -338,8 +654,13 @@ export class Networks { async importFromLISHnet(data: ILISHNetwork, enabled: boolean = false): Promise { const definition = this.validateNetwork(data); const config: LISHNetworkConfig = { ...definition, enabled }; - upsertLISHnet(this.db, config.networkID, config.name, config.description, config.bootstrapPeers, config.enabled, config.created); - if (enabled) await this.joinNetwork(config.networkID); + // An upsert can bring a network into existence — see {@link catalogMutex}. + const previous = await this.inCatalog(() => { + const row = this.get(config.networkID); + upsertLISHnet(this.db, config.networkID, config.name, config.description, config.bootstrapPeers, config.enabled, config.created); + return row; + }); + await this.reconcile(config.networkID, previous); return config; } @@ -379,33 +700,104 @@ export class Networks { return listEnabledLISHnets(this.db); } - add(network: LISHNetworkConfig): boolean { - return addLISHnet(this.db, network); - } - - update(network: LISHNetworkConfig): boolean { - return updateLISHnet(this.db, network); + async add(network: LISHNetworkConfig): Promise { + const ok = await this.inCatalog(() => addLISHnet(this.db, network)); + // A network added as enabled has to be joined, not merely written down. An add of one + // that already exists writes nothing and reconciles nothing: it used to claim a + // revision anyway on the way in, which cancelled a queued enable of that very network + // — a request that changed nothing discarding one that meant something. + if (ok) await this.reconcile(network.networkID, undefined); + return ok; + } + + async update(network: LISHNetworkConfig): Promise { + // Row read and write in ONE critical section. With the read outside it, a toggle + // could slip between the two and be overwritten by a row this edit had already read. + const staged = await this.inCatalog(() => { + const existing = this.get(network.networkID); + // Store the cleaned list, not the raw one: blank rows from the form would + // otherwise be persisted while the runtime worked from the filtered copy, and + // the two would disagree about what this network's bootstrap list even is. + const cleaned = Networks.cleanBootstrapList(network.bootstrapPeers ?? []); + return updateLISHnet(this.db, { ...network, bootstrapPeers: cleaned }) ? { existing } : undefined; + }); + if (!staged) return false; + // The general edit form carries the bootstrap list AND the enabled flag, so this path + // can change either one. Without the runtime reconciliation the edit would reach only + // the database and the live node would keep dialing the previous list — or stay in a + // network the edit had just disabled — until restart. + await this.reconcile(network.networkID, staged.existing); + return true; } + /** + * Delete a lishnet: drop the row, then leave it. + * + * The row goes first so the reconcile converges on the only desired state a deleted + * lishnet has — no row, therefore not joined — and an enable arriving during the leave + * finds no row and answers "not found" instead of rejoining a deleted network. Both + * halves used to be separate lock acquisitions with the row write LAST, which let that + * enable rejoin the topic between them and then watch the delete remove the row + * underneath it: subscribed, in `joinedNetworks`, and nothing in the database to explain + * either. + */ async delete(id: string): Promise { - await this.setEnabled(id, false); - return deleteLISHnet(this.db, id); + const staged = await this.inCatalog(() => { + if (!lishnetExists(this.db, id)) return undefined; + const previous = this.get(id); + deleteLISHnet(this.db, id); + return { previous }; + }); + if (!staged) return false; + await this.reconcile(id, staged.previous); + return true; } exists(id: string): boolean { return lishnetExists(this.db, id); } - addIfNotExists(network: LISHNetworkDefinition): boolean { - return addLISHnetIfNotExists(this.db, network); + /** + * Add one definition if it does not exist yet. Nothing to reconcile — the writer inserts + * it DISABLED — but the insert itself still belongs under {@link catalogMutex}: it went + * straight to the database, so a `replace()` that had already read the catalog could + * delete the row this had just reported as added, and the ID set `replace()` computes + * its affected list from moved underneath it. + */ + async addIfNotExists(network: LISHNetworkDefinition): Promise { + return await this.inCatalog(() => addLISHnetIfNotExists(this.db, network)); } - importNetworks(networks: LISHNetworkDefinition[]): number { - return importLISHnets(this.db, networks); + /** + * Add every definition that does not exist yet, as one batch under the catalog mutex. + * Nothing to reconcile: the underlying writer skips networks that already exist and + * inserts new ones DISABLED, so no network's runtime state can change here — but see + * {@link addIfNotExists} for why the write is still not the caller's to do unlocked. + */ + async importNetworks(networks: LISHNetworkDefinition[]): Promise { + return await this.inCatalog(() => importLISHnets(this.db, networks)); } - replace(networks: LISHNetworkConfig[]): void { - replaceLISHnets(this.db, networks); + /** + * Replace the whole stored list (used for reordering). Every network the write touched + * is reconciled afterwards, including ones it dropped: a wholesale rewrite can enable, + * disable, re-list or delete anything, and a deleted network that is still joined would + * otherwise stay in its topic with no row left to explain it. + */ + async replace(networks: LISHNetworkConfig[]): Promise { + // Snapshot and rewrite in one critical section, so the list this reconciles against + // is exactly the list it replaced. Reading it outside meant a network created while + // we waited was rewritten out of existence behind the back of the add that was still + // joining it. + const before = await this.inCatalog(() => { + const rows = new Map(this.list().map(n => [n.networkID, n])); + replaceLISHnets(this.db, networks); + return rows; + }); + // One lishnet at a time, each under its own lock and none of them under the catalog. + // Reconciling the whole set under the global lock meant a rewrite of a long list held + // it across every affected network's dials and disconnects in turn. + for (const id of new Set([...before.keys(), ...networks.map(n => n.networkID)])) await this.reconcile(id, before.get(id)); } /** @@ -429,26 +821,63 @@ export class Networks { * recorded. Returns the updated config or null if the network is unknown. */ async updateBootstrapPeers(id: string, bootstrapPeers: string[]): Promise { - const existing = this.get(id); - if (!existing) return null; - const cleaned = bootstrapPeers.filter(p => typeof p === 'string' && p.trim().length > 0); - // Drop the bootstrap-exemption for peer IDs removed from this network's - // config, unless still configured for another joined network. Prevents a - // removed bootstrap entry from lingering as infrastructure that a later - // leave-network would refuse to disconnect. + const staged = await this.inCatalog(() => { + const existing = this.get(id); + if (!existing) return undefined; + const next: LISHNetworkConfig = { ...existing, bootstrapPeers: Networks.cleanBootstrapList(bootstrapPeers) }; + // Persist first and believe the answer. Switching the runtime over after a failed + // write would leave the node dialing a list the database never accepted, and the + // old one would come back at the next restart with nothing to explain the change. + if (!updateLISHnet(this.db, next)) throw new CodedError(ErrorCodes.NETWORK_NOT_FOUND, id); + return { existing, next }; + }); + if (!staged) return null; + await this.reconcile(id, staged.existing); + return staged.next; + } + + /** + * The same normalisation the repository applies on write, for the comparisons here + * that have to speak about a list in the shape it will be stored in. + */ + private static cleanBootstrapList(peers: string[]): string[] { + return cleanBootstrapList(peers); + } + + /** + * Bring the running node in line with a network's new configured bootstrap list. + * + * Shared by the bootstrap-only editor and the general network edit form, because + * both can change that list and a change that reaches only the database leaves the + * live node working from the previous one until restart. + * + * Three things have to happen: peer IDs that left the list lose their + * bootstrap-exemption (unless another joined network still configures them, else a + * removed entry lingers as infrastructure a later leave refuses to disconnect), + * the status rows are pruned — which also invalidates any bootstrap job still + * walking the old list — and the new entries are dialed. + */ + private syncBootstrapRuntime(id: string, previousPeers: string[], cleaned: string[]): void { const nextIDs = new Set(Networks.bootstrapPeerIDsOf(cleaned)); const elsewhere = this.configuredBootstrapPeerIDsElsewhere(id); - for (const pid of Networks.bootstrapPeerIDsOf(existing.bootstrapPeers)) { + for (const pid of Networks.bootstrapPeerIDsOf(previousPeers)) { if (!nextIDs.has(pid) && !elsewhere.has(pid)) this.network.pruneConfiguredBootstrapPeer(pid); } - const next: LISHNetworkConfig = { ...existing, bootstrapPeers: cleaned }; - updateLISHnet(this.db, next); + // Addresses that left the list while their peer ID stayed — the user edited a + // host or port. The identity-level prune above cannot see those, so recovery + // would go on dialing the address that was replaced. + // Compare canonically, the same way the autodial list itself does. Raw string + // equality would treat two spellings of one address (DNS case, IPv6 form) as + // different entries here and as the same one during the prune below. + const keptAddresses = new Set(cleaned.map(normalizeMultiaddrForCompare)); + const elsewhereAddresses = this.configuredBootstrapAddressesElsewhere(id); + const dropped = Networks.cleanBootstrapList(previousPeers).filter(a => !keptAddresses.has(normalizeMultiaddrForCompare(a)) && !elsewhereAddresses.has(normalizeMultiaddrForCompare(a))); + this.network.pruneBootstrapAddresses(dropped); this.network.pruneBootstrapStatus(id, cleaned); if (this.joinedNetworks.has(id) && cleaned.length > 0) { this.network.addBootstrapPeers(cleaned, id, 'configured').catch(err => { - console.error(`[Networks] re-dial after updateBootstrapPeers failed:`, err?.message ?? err); + console.error(`[Networks] bootstrap re-dial after config change failed:`, err?.message ?? err); }); } - return next; } } diff --git a/backend/src/protocol/bootstrap-status.ts b/backend/src/protocol/bootstrap-status.ts index e0356da0a..0cd6c28b3 100644 --- a/backend/src/protocol/bootstrap-status.ts +++ b/backend/src/protocol/bootstrap-status.ts @@ -1,10 +1,63 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialStatus, type BootstrapPeerOrigin } from '@shared'; +import { canonicalMultiaddr } from './multiaddr-utils.ts'; + +/** + * Hard ceiling on DISCOVERED (gossip-learned) rows kept per network. A hostile + * topic subscriber can announce unbounded unique addresses, all ending in one + * connected peer's ID; without a cap the tracker map, its snapshots, and the + * WebSocket updates grow without limit. Configured rows are never counted or + * dropped — they are finite user data. + */ +const MAX_DISCOVERED_PER_NETWORK = 256; + +/** + * While a {@link BootstrapStatusTracker.batchDebounced} frame is open, how often the + * changes accumulated so far are published. + * + * A plain batch frame publishes once, at close — which is wrong for a body that awaits a + * dial per address: the frame would stay open for the whole list, one 10 s timeout at a + * time, and the UI would show nothing until the last address settled. Flushing on this + * interval keeps the run visible while still collapsing the burst of intake mutations + * (two per address, each rebuilding the whole snapshot) into a handful of emissions. + */ +const BATCH_FLUSH_INTERVAL_MS = 75; + +/** + * A stored row: the status the UI sees, plus the clock {@link BootstrapStatusTracker.sweepStale} + * measures against. + * + * `updatedAt` cannot serve as that clock. It is "when did anything about this row last + * change", which is what the UI wants, and it moves on every dial outcome — failures + * included. Gossip re-announces a dead peer far more often than the sweep TTL, and this + * node answers each mention with a dial that fails, so measuring staleness from + * `updatedAt` meant a dead row was refreshed by its own failures and could never expire. + * `staleSince` moves only when the address actually answered. + */ +type TrackedPeer = BootstrapPeerStatus & { staleSince: number }; + +/** + * Which spelling of an endpoint the UI should show. + * + * The row is keyed canonically, so several spellings can land on it. A configured one + * always wins — it is what the user typed into the form and what they will look for when + * fixing it — and otherwise the first spelling seen is kept, so a row does not visibly + * change address every time gossip restates it differently. + */ +function displaySpelling(previous: TrackedPeer | undefined, incoming: string, origin: BootstrapPeerOrigin): string { + if (!previous || origin === 'configured') return incoming; + return previous.multiaddr; +} /** * Tracks per-network, per-bootstrap-peer dial outcome status. * - * Outer key is networkID; inner key is the exact multiaddr string from the network - * config. Populated by markBootstrapPending / recordBootstrapOutcome when called + * Outer key is networkID; inner key is the CANONICAL form of the multiaddr, while the + * row keeps the spelling as written for display. Keying by the raw string let two + * spellings of one endpoint — DNS case, a trailing dot, an expanded IPv6 literal — open + * two rows that then contradicted each other, spent the row budget twice and survived a + * delete aimed at only one of them. + * + * Populated by markBootstrapPending / recordBootstrapOutcome when called * with a networkID context. Lets the UI surface which SPECIFIC bootstrap entry is * stale (identity-mismatch) or unreachable (timeout), rather than flagging the * whole network. @@ -15,16 +68,137 @@ import { type BootstrapStatus, type BootstrapPeerStatus, type BootstrapPeerDialS * under the network through which they were learned. */ export class BootstrapStatusTracker { - private readonly stats: Map> = new Map(); + private readonly stats: Map> = new Map(); private onStatusChange: ((networkID: string, status: BootstrapStatus) => void) | null = null; + /** Open {@link batch} frames, keyed by networkID. See that method for why. */ + private readonly batches: Map = new Map(); + /** Current members of a network, for {@link capDiscovered}. See {@link setMembersProvider}. */ + private membersProvider: ((networkID: string) => Set) | null = null; /** Register a callback that fires on every status mutation. */ setOnChange(cb: ((networkID: string, status: BootstrapStatus) => void) | null): void { this.onStatusChange = cb; } + /** + * Supply the current member set of a network, so {@link capDiscovered} can tell a row + * that describes a live participant from one that describes an address nobody answers + * on. Asked for the whole set rather than per peer because the cap ranks every row at + * once and a per-peer question would rebuild the same snapshot for each of them. + */ + setMembersProvider(fn: ((networkID: string) => Set) | null): void { + this.membersProvider = fn; + } + + /** + * Group many mutations of one network into a SINGLE status emission. + * + * Every mutation otherwise rebuilds and emits the whole peer list, and a caller + * that walks a list of addresses performs two of them per address (pending, then + * outcome) — so intake of one large announce costs a snapshot per row per address, + * each copying every row, all but the last of which is thrown away by the UI. + * + * The frame closes on every exit path, throw included, so a body that fails still + * publishes what it managed to change — the tracker is already mutated by then and + * silence would leave the UI showing pre-batch state indefinitely. An async body is + * held open until its promise settles, because the caller this exists for awaits a + * dial between mutations; the return value keeps the body's own type either way. + * Nested calls for the same network collapse into the outermost frame, and a batch + * in which nothing actually changed emits nothing. + */ + batch(networkID: string, fn: () => T): T { + let frame = this.batches.get(networkID); + if (!frame) { + frame = { depth: 0, dirty: false }; + this.batches.set(networkID, frame); + } + frame.depth++; + const open = frame; + let result: T; + try { + result = fn(); + } catch (err) { + this.closeBatch(networkID, open); + throw err; + } + if (result instanceof Promise) { + return result.then( + value => { + this.closeBatch(networkID, open); + return value; + }, + err => { + this.closeBatch(networkID, open); + throw err; + } + ) as T; + } + this.closeBatch(networkID, open); + return result; + } + + /** + * Like {@link batch}, but for an async body long enough that holding every change to + * the end would leave the UI stale: what has accumulated is published every + * {@link BATCH_FLUSH_INTERVAL_MS} while the frame is open, and once more at close. + * + * This is the shape bootstrap intake needs — it awaits a dial between each address's + * pending mark and its outcome, so the alternatives are one emission per mutation + * (what it used to do) or one emission for the entire list (a frozen UI). + */ + async batchDebounced(networkID: string, fn: () => Promise): Promise { + let frame = this.batches.get(networkID); + if (!frame) { + frame = { depth: 0, dirty: false }; + this.batches.set(networkID, frame); + } + frame.depth++; + const open = frame; + const timer = setInterval(() => { + // A frame replaced by clear() belongs to a torn-down run; publishing its + // leftovers under the same networkID would speak for whatever came next. + if (!open.dirty || this.batches.get(networkID) !== open) return; + open.dirty = false; + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + }, BATCH_FLUSH_INTERVAL_MS); + timer.unref?.(); + try { + return await fn(); + } finally { + clearInterval(timer); + this.closeBatch(networkID, open); + } + } + + /** Leave one {@link batch} frame, emitting the grouped snapshot when the last one exits. */ + private closeBatch(networkID: string, frame: { depth: number; dirty: boolean }): void { + frame.depth--; + if (frame.depth > 0) return; + // clear() drops open frames on teardown; a pending emission from before it + // belongs to the run that was torn down, not to whatever comes next. + if (this.batches.get(networkID) !== frame) return; + this.batches.delete(networkID); + if (frame.dirty) this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + + /** + * Publish one network's current status, or defer to the end of the open batch. + * + * Deferring skips {@link buildStatus} as well as the callback — building the + * snapshot is the part that copies every row, so suppressing only the callback + * would leave the cost in place. + */ + private notify(networkID: string): void { + const frame = this.batches.get(networkID); + if (frame) { + frame.dirty = true; + return; + } + this.onStatusChange?.(networkID, this.buildStatus(networkID) ?? { networkID, peers: [] }); + } + /** Iterate over all tracked network IDs and their peer maps. Used for NET-CHURN dump. */ - entries(): IterableIterator<[string, Map]> { + entries(): IterableIterator<[string, Map]> { return this.stats.entries(); } @@ -42,63 +216,214 @@ export class BootstrapStatusTracker { markPending(networkID: string | null, multiaddr: string, expectedPeerID: string | null, origin: BootstrapPeerOrigin): void { if (!networkID) return; const net = this.ensureNetwork(networkID); + const key = canonicalMultiaddr(multiaddr); // Preserve a stronger origin classification — once we know an entry is in // the saved config ('configured'), an inbound peer-announce later restating // the same multiaddr must not downgrade it to 'discovered'. - const previous = net.get(multiaddr); + const previous = net.get(key); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; - net.set(multiaddr, { multiaddr, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: null, lastError: null, updatedAt: new Date().toISOString() }); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + const display = displaySpelling(previous, multiaddr, origin); + // Keep the existing staleness clock (see sweepStale). Reaching this point means + // someone MENTIONED the peer again — gossip repeating an address it still + // remembers — which is evidence about the announcer, not about the peer. Letting + // a mention move the clock made a dead peer's row immortal: every announce cycle + // is far shorter than the sweep TTL, so the row was refreshed long before it + // could expire, no matter how many dials to it had already failed. Only a dial + // that actually CONNECTED advances it, in recordOutcome below. + // Keep any identity a previous dial actually PROVED on this endpoint. Clearing it + // here threw away the one piece of evidence the row-cap ranking trusts, and gossip + // re-mentions an address constantly — so a verified member's row was demoted to an + // ordinary one within an announce cycle of being verified. A new dial result + // overwrites it in recordOutcome; only that can change who is behind the address. + net.set(key, { multiaddr: display, expectedPeerID, status: 'pending', origin: finalOrigin, actualPeerID: previous?.actualPeerID ?? null, lastError: null, updatedAt: previous?.updatedAt ?? new Date().toISOString(), staleSince: previous?.staleSince ?? Date.now() }); + this.capDiscovered(networkID, net); + this.notify(networkID); } /** Record a dial outcome (connected, timeout, error, identity-mismatch). */ recordOutcome(networkID: string | null, multiaddr: string, expectedPeerID: string | null, status: BootstrapPeerDialStatus, message: string | null, actualPeerID: string | null, origin: BootstrapPeerOrigin): void { if (!networkID) return; const net = this.ensureNetwork(networkID); + const key = canonicalMultiaddr(multiaddr); const truncated = message ? (message.length > 200 ? message.slice(0, 200) + '…' : message) : null; - const previous = net.get(multiaddr); + const previous = net.get(key); const finalOrigin: BootstrapPeerOrigin = previous?.origin === 'configured' ? 'configured' : origin; - net.set(multiaddr, { multiaddr, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString() }); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + const display = displaySpelling(previous, multiaddr, origin); + // Only success restarts the staleness clock. A FAILING outcome is the node's own + // reaction to somebody else's mention of a dead peer, so letting it advance the + // clock kept exactly the rows this sweep exists to remove: gossip mentions the + // peer, the dial fails, the row is refreshed, and the TTL is never reached. + net.set(key, { multiaddr: display, expectedPeerID, status, origin: finalOrigin, actualPeerID, lastError: truncated, updatedAt: new Date().toISOString(), staleSince: status === 'connected' ? Date.now() : (previous?.staleSince ?? Date.now()) }); + this.capDiscovered(networkID, net); + this.notify(networkID); + } + + /** + * Mark every row for one endpoint reachable, in every network that has one. + * + * For the loops that probe an ADDRESS rather than a network's list: they know the + * endpoint answered but not who was waiting to hear it. The parked-bootstrap probe is + * the case that matters — it is the only thing that ever retries an address the + * routability filter rejected at configure time, and it used to keep its success to + * itself, so the row stayed red for an address that had been working for hours. + * + * Matching is canonical, not by string identity: the probe walks parsed multiaddrs + * while the rows are keyed by the spelling the user typed. + */ + recordAddressReachable(address: string): void { + const target = canonicalMultiaddr(address); + for (const [networkID, peers] of this.stats) { + const peer = peers.get(target); + if (!peer || peer.status === 'connected') continue; + // The probe knows the endpoint answered, not who answered — so it neither sets + // nor clears the verified identity a real dial may already have established. + peers.set(target, { ...peer, status: 'connected', lastError: null, updatedAt: new Date().toISOString(), staleSince: Date.now() }); + this.notify(networkID); + } + } + + /** + * Bound discovered rows per network — see MAX_DISCOVERED_PER_NETWORK. + * + * Age alone is the wrong order. It looks at neither the row's state nor whether the + * peer is actually in the network, so a flood of freshly invented dead addresses could + * push a live, connected participant out of the list — the stale sweep protects an + * active member deliberately, and this used to undo that. Least useful goes first: + * a row whose address failed, then one that has never answered, then one that once + * connected, and rows belonging to a VERIFIED member last. + * + * Verified is the operative word. Ranking on `expectedPeerID` — the identity the + * ADDRESS claims — handed the protection to whoever was making the claim: invented + * addresses that all named a live member each took the top rank, and the member's own + * genuine row, being the oldest of them, was the one evicted. Only `actualPeerID`, set + * from a connection we actually made, is evidence of anything. + */ + private capDiscovered(networkID: string, net: Map): void { + let discovered = 0; + for (const p of net.values()) if (p.origin === 'discovered') discovered++; + if (discovered <= MAX_DISCOVERED_PER_NETWORK) return; + const members = this.membersProvider?.(networkID) ?? new Set(); + const rankOf = (p: TrackedPeer): number => { + if (p.actualPeerID && members.has(p.actualPeerID)) return 3; + if (p.status === 'connected') return 2; + if (p.status === 'pending') return 1; + return 0; + }; + // Ranked once per row, not inside the comparator — the member lookup would + // otherwise run O(n log n) times over the same snapshot. + const victims = [...net.entries()] + .filter(([, p]) => p.origin === 'discovered') + .map(([key, p]) => ({ key, rank: rankOf(p), age: Date.parse(p.updatedAt) })) + .sort((a, b) => a.rank - b.rank || a.age - b.age); + for (let i = 0; i < discovered - MAX_DISCOVERED_PER_NETWORK; i++) net.delete(victims[i]!.key); } /** Drop a single peer entry directly (used after identity-mismatch purge of discovered peers). */ deletePeer(networkID: string, multiaddr: string): void { const net = this.stats.get(networkID); if (!net) return; - net.delete(multiaddr); + net.delete(canonicalMultiaddr(multiaddr)); if (net.size === 0) this.stats.delete(networkID); - const snap = this.buildStatus(networkID) ?? { networkID, peers: [] }; - this.onStatusChange?.(networkID, snap); + this.notify(networkID); + } + + /** + * Drop every discovered-origin entry recorded for the given peer identity, in + * every network. Used when a peer is evicted as unreachable — its gossip-learned + * rows are pure noise at that point. Configured rows are kept: they are user + * data and must stay visible (red) so the user can fix or remove them. + */ + deleteDiscoveredByPeerID(peerID: string): void { + for (const [networkID, peers] of [...this.stats]) { + let changed = false; + for (const [addr, p] of [...peers]) { + if (p.origin !== 'discovered') continue; + if (p.expectedPeerID !== peerID && p.actualPeerID !== peerID) continue; + peers.delete(addr); + changed = true; + } + if (!changed) continue; + if (peers.size === 0) this.stats.delete(networkID); + this.notify(networkID); + } + } + + /** + * Drop discovered-origin entries that have gone stale: nothing has CONNECTED on the + * address within `ttlMs` AND the peer is not an active member of THAT network. A peer + * that dies stops answering, so its clock freezes and the row expires here — whether + * gossip keeps naming it or not, and whether the row is frozen at 'connected' or + * cycling through failures that this node produces itself. See {@link TrackedPeer}. The + * liveness predicate is scoped to the network (its topic subscribers), NOT the + * shared libp2p connection: a peer that left network B but is still connected + * through network A must not keep a stale row under B. Membership is judged on the + * VERIFIED identity only — see the check below. Configured entries are exempt (user + * data). `now` is injectable for tests. + */ + sweepStale(ttlMs: number, isMember: (networkID: string, peerID: string) => boolean, now: number = Date.now()): void { + for (const [networkID, peers] of [...this.stats]) { + let changed = false; + for (const [addr, p] of [...peers]) { + if (p.origin !== 'discovered') continue; + // Only a VERIFIED identity exempts a row. `expectedPeerID` is whatever the + // address claims, and a discovered multiaddr practically always carries one — + // so reading it here handed the exemption to the announcer: any address ending + // /p2p/ was treated as that member's, never dialed successfully, + // and never expired. The cap bounds how many such rows exist; this is what + // stops them from occupying the budget permanently. + if (p.actualPeerID && isMember(networkID, p.actualPeerID)) continue; + if (now - p.staleSince < ttlMs) continue; + peers.delete(addr); + changed = true; + } + if (!changed) continue; + if (peers.size === 0) this.stats.delete(networkID); + this.notify(networkID); + } } - /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ + /** + * Drop bootstrap status entries no longer in the configured peer list (after an update). + * + * `keepMultiaddrs` is the network's CONFIGURED list, so only configured rows may be + * judged by it. Discovered rows are not in it and never will be — deleting them here + * would clear the participant list of everything gossip has found, on nothing more + * than a bootstrap edit or a "refresh from public list", until gossip happens to + * mention each peer again. Discovered rows leave via their own paths: the staleness + * sweep, the per-network cap, or eviction of the peer ID. + */ pruneEntries(networkID: string, keepMultiaddrs: string[]): void { const peers = this.stats.get(networkID); if (!peers) return; - const keep = new Set(keepMultiaddrs); - for (const addr of [...peers.keys()]) { - if (!keep.has(addr)) peers.delete(addr); + // Canonical, like the keys themselves: a configured entry re-typed in a different + // but equivalent spelling is the same entry, not a removed one. + const keep = new Set(keepMultiaddrs.map(canonicalMultiaddr)); + for (const [addr, peer] of [...peers.entries()]) { + if (peer.origin === 'configured' && !keep.has(addr)) peers.delete(addr); } if (peers.size === 0) this.stats.delete(networkID); - const snapshot = this.buildStatus(networkID); - if (snapshot) this.onStatusChange?.(networkID, snapshot); + // Emit the empty list rather than nothing when the last row goes: buildStatus + // returns null for a dropped network, and skipping the callback would leave the + // UI showing the very row that was just removed. Same fallback the other + // removal paths use. + this.notify(networkID); } /** Reset the bootstrap status for a single network (used when re-joining). */ resetNetwork(networkID: string): void { this.stats.delete(networkID); - this.onStatusChange?.(networkID, { networkID, peers: [] }); + this.notify(networkID); } /** Clear all tracked state (called from Network.stop()). */ clear(): void { this.stats.clear(); + // An in-flight batch belongs to the run being torn down; its pending emission + // would publish the next run's (empty) state under the old run's networkID. + this.batches.clear(); } - private ensureNetwork(networkID: string): Map { + private ensureNetwork(networkID: string): Map { let net = this.stats.get(networkID); if (!net) { net = new Map(); @@ -110,6 +435,7 @@ export class BootstrapStatusTracker { private buildStatus(networkID: string): BootstrapStatus | null { const peers = this.stats.get(networkID); if (!peers) return null; - return { networkID, peers: [...peers.values()].map(p => ({ ...p })) }; + // staleSince is internal bookkeeping, not part of the wire contract. + return { networkID, peers: [...peers.values()].map(({ staleSince: _staleSince, ...p }) => p) }; } } diff --git a/backend/src/protocol/multiaddr-utils.ts b/backend/src/protocol/multiaddr-utils.ts new file mode 100644 index 000000000..8c41155ab --- /dev/null +++ b/backend/src/protocol/multiaddr-utils.ts @@ -0,0 +1,85 @@ +/** + * Multiaddr helpers shared by the runtime network layer and the libp2p config + * builder. + * + * They live in their own module because `network-config.ts` runs before, and is + * imported by, `network.ts` — pulling the whole network module in just to reach a + * two-line helper would make that cycle. Everything here is pure. + */ + +import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; + +/** Multiaddr component code of a `/p2p/` segment. */ +export const MULTIADDR_P2P_CODE = 421; + +/** + * Peer ID we would actually connect to when dialing a multiaddr, or null when the + * address carries no `/p2p` component. + * + * A relayed address (`.../p2p//p2p-circuit/p2p/`) carries two `/p2p` + * components; the destination — the peer whose identity the Noise handshake verifies + * — is the LAST one. Taking the first returns the relay instead, which then gets + * protected, tagged or suppressed in place of the peer actually meant. + */ +export function extractDestinationPeerID(ma: any): string | null { + try { + const components = ma.getComponents().filter((c: { code: number }) => c.code === MULTIADDR_P2P_CODE); + return components.length > 0 ? (components[components.length - 1].value ?? null) : null; + } catch { + return null; + } +} + +/** Same as {@link extractDestinationPeerID} but taking the address as a string. */ +export function destinationPeerIDOf(address: string): string | null { + try { + return extractDestinationPeerID(Multiaddr(address.trim())); + } catch { + return null; + } +} + +/** + * Bounded memo for the canonical form. Canonicalising parses the address, and callers + * compare one address against the whole bootstrap list, so the same handful of strings + * would be re-parsed constantly. The cap stops a flood of gossip-invented addresses + * from turning the cache into a leak — it is only a cache, so clearing it wholesale + * costs a recompute and nothing else. + */ +const CANONICAL_CACHE_LIMIT = 4096; +const canonicalCache = new Map(); + +/** + * One spelling per address, for comparing two multiaddrs that mean the same thing. + * + * The string goes through the multiaddr parser first, which is what collapses an + * expanded IPv6 literal to its compressed form and settles component ordering — a regex + * over the raw text cannot do that, so two spellings of one address would count as two + * different entries. Only then is the DNS host folded: DNS is case-insensitive and may + * carry the FQDN root dot, while a base58 peer ID in the same string is + * case-SIGNIFICANT and must survive untouched. + * + * An unparseable input comes back trimmed rather than throwing: callers use this for + * equality between values they already hold, and "compares equal only to itself" is the + * safe answer there. + */ +export function canonicalMultiaddr(address: string): string { + const cached = canonicalCache.get(address); + if (cached !== undefined) return cached; + const result = computeCanonicalMultiaddr(address); + if (canonicalCache.size >= CANONICAL_CACHE_LIMIT) canonicalCache.clear(); + canonicalCache.set(address, result); + return result; +} + +/** Uncached canonicalisation — see {@link canonicalMultiaddr}. */ +function computeCanonicalMultiaddr(address: string): string { + const trimmed = address.trim(); + let parsed = trimmed; + try { + parsed = Multiaddr(trimmed).toString(); + } catch { + // keep the trimmed original + } + return parsed.replace(/\/(dns|dns4|dns6|dnsaddr)\/([^/]+)/gi, (_match, protocol: string, host: string) => `/${protocol.toLowerCase()}/${host.toLowerCase().replace(/\.+$/, '')}`); +} diff --git a/backend/src/protocol/network-config.ts b/backend/src/protocol/network-config.ts index 555940ce3..a6842345d 100644 --- a/backend/src/protocol/network-config.ts +++ b/backend/src/protocol/network-config.ts @@ -25,6 +25,7 @@ import { trace } from '../logger.ts'; import { normalizeTrustedPeerIds, parseAcceptPXThreshold } from './constants.ts'; import { getLocalCidrs, shouldDenyDial, extractFirstIPv4 } from './address-filter.ts'; import { peerIdFromString } from '@libp2p/peer-id'; +import { extractDestinationPeerID, destinationPeerIDOf } from './multiaddr-utils.ts'; const { multiaddr: Multiaddr } = await import('@multiformats/multiaddr'); /** A gossipsub direct-peer entry: a peer id and its multiaddrs. */ @@ -45,7 +46,7 @@ function buildDirectPeersFromBootstrap(uniquePeers: string[]): DirectPeer[] { for (const ma of uniquePeers) { try { const parsed = Multiaddr(ma); - const pid = parsed.getComponents().find((c: any) => c.code === 421)?.value; + const pid = extractDestinationPeerID(parsed); if (!pid) continue; direct.push({ id: peerIdFromString(pid), addrs: [parsed] }); } catch { @@ -74,7 +75,10 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult const bootstrapMultiaddrs: any[] = []; // Unique bootstrap peers computed up-front so gossipsub config below can // pre-populate directPeers from them. - const uniqueBootstrapPeers = [...new Set(bootstrapPeers)].filter(p => !p.includes(myPeerID)); + // Self is decided by the DESTINATION identity, not by "the string mentions us". A + // relayed entry `/…/p2p//p2p-circuit/p2p/` names us as the RELAY hop while + // targeting somebody else, and the substring test threw it away as our own address. + const uniqueBootstrapPeers = [...new Set(bootstrapPeers)].filter(p => destinationPeerIDOf(p) !== myPeerID); const peerExchange = allSettings.network?.peerExchange; const pxEnabled = peerExchange?.enabled === true; const parsedThreshold = parseAcceptPXThreshold(peerExchange?.acceptPXThreshold); @@ -210,8 +214,7 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult // trusted peers blocked when their advertised addr lives on a LAN segment // different from our own. Trusted peers are by policy known-good // destinations, so dial them regardless of CIDR match. - const pidComponent = ma?.getComponents?.()?.find?.((c: any) => c.code === 421); - const pid = pidComponent?.value ?? null; + const pid = extractDestinationPeerID(ma); if (pid && (bootstrapPeerIDs.has(pid) || trustedPXPeerIDs.has(pid))) return false; const deny = shouldDenyDial(ma, getLocalCidrs()); if (deny) { @@ -410,7 +413,7 @@ export function buildLibp2pConfig(params: BuildConfigParams): BuildConfigResult console.log(' -', peer); try { const ma = Multiaddr(peer); - const peerID = ma.getComponents().find(c => c.code === 421)?.value ?? null; + const peerID = extractDestinationPeerID(ma); if (peerID) { bootstrapPeerIDs.add(peerID); bootstrapMultiaddrs.push(ma); diff --git a/backend/src/protocol/network.ts b/backend/src/protocol/network.ts index d3dad5566..f8f42064c 100644 --- a/backend/src/protocol/network.ts +++ b/backend/src/protocol/network.ts @@ -1,4 +1,5 @@ import { createLibp2p } from 'libp2p'; +import { Mutex } from 'async-mutex'; import { KEEP_ALIVE } from '@libp2p/interface'; import { SqliteDatastore } from './datastore.ts'; import { privateKeyToProtobuf } from '@libp2p/crypto/keys'; @@ -15,6 +16,7 @@ import { buildLibp2pConfig } from './network-config.ts'; import { type WantMessage } from './downloader.ts'; import { lishTopic, LISH_TOPIC_PREFIX } from './constants.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; +import { canonicalMultiaddr, extractDestinationPeerID } from './multiaddr-utils.ts'; import { CodedError, ErrorCodes, type NetworkNodeInfo, type PeerConnectionInfo, type IMeshHealth, type BootstrapStatus, type BootstrapPeerDialStatus, type BootstrapPeerOrigin } from '@shared'; import { Circuit } from '@multiformats/multiaddr-matcher'; import { createTopicScoreParams } from '@chainsafe/libp2p-gossipsub/score'; @@ -91,6 +93,118 @@ const WANT_RESPONSE_COOLDOWN_MS = 60_000; const WANT_RESPONSE_CLEANUP_INTERVAL_MS = 5 * 60_000; /** Search query dedup window — same `searchID` arriving via mesh within this period is ignored. */ const SEARCH_DEDUP_TTL_MS = 5 * 60_000; +/** + * Consecutive re-dial failures after which a peer is treated as gone and evicted + * (peerStore + bootstrap sets + its discovered status rows). Combined with + * REDIAL_EVICT_MIN_MS so a burst of quick failures right after our own restart + * or a network partition cannot mass-purge peers that are merely slow to return. + */ +const REDIAL_EVICT_FAILS = 6; +/** Minimum continuous unreachability (since the first recorded failure) before eviction. */ +const REDIAL_EVICT_MIN_MS = 30 * 60_000; +/** + * Discovered bootstrap-status rows older than this (and without a live connection) + * are dropped from the UI. Live peers keep refreshing their rows via gossip intake; + * dead ones stop being mentioned, freeze, and expire here. + */ +const BOOTSTRAP_STATUS_STALE_MS = 30 * 60_000; +/** + * How long an evicted-as-unreachable peer stays quarantined in addBootstrapPeers. + * Gossip from nodes that still remember the dead peer keeps mentioning it; without + * this window every mention would re-create its status row and burn a dial. Once + * the window lapses a single probe is allowed again (self-heals on peer return). + * + * Kept equal to BOOTSTRAP_STATUS_STALE_MS deliberately: shorter would let stale + * gossip refresh rows faster than the sweep can expire them; longer would only + * delay re-discovery of a peer that genuinely came back (a returned peer that + * dials US escapes immediately via the peer:connect reset — this window matters + * only for peers that cannot initiate inbound connections). + */ +const UNREACHABLE_QUARANTINE_MS = 30 * 60_000; + +/** + * Backoff ceiling for the loops that probe ONE CONFIGURED ADDRESS. + * + * A configured entry is exempt from eviction and from quarantine — it is user data and + * the way back into the network — but "never given up on" is not "dialed without limit". + * Each attempt spends a 10 s dial timeout, so a handful of dead configured addresses + * could occupy a status tick end to end, every tick, forever. + * + * Half the general re-dial ceiling (10 min) deliberately: a configured address deserves + * to be retried more often than a gossip-learned one, so a bootstrap that comes back is + * picked up within five minutes without the operator touching anything, while a + * permanently dead one costs one dial per five minutes instead of one per 30 s tick. + */ +const CONFIGURED_PROBE_BACKOFF_MAX_MS = 5 * 60_000; + +/** + * Ceiling on the autodial list zero-connection recovery walks. + * + * A discovered address earns its place by answering a dial, so the list grows with the + * number of distinct endpoints that have ever worked — unbounded on a fleet with churn, + * and the array is otherwise only ever shortened by an identity purge. Over the ceiling + * the OLDEST DISCOVERED entry goes: configured entries are finite user data and the way + * back into a network, so they are never the ones dropped. 512 is far above any real + * node's working set while keeping the list, and the walk over it, bounded. + */ +const MAX_BOOTSTRAP_ADDRESSES = 512; + +/** + * Where the eviction window should run from after a re-dial failure. + * + * A failure only says something about the PEER when this node can reach anyone + * at all. While we are the disconnected one — laptop asleep, Wi-Fi off, VPN + * dropped — every dial fails, so the window is slid forward instead of + * accumulating. Without this, a local outage longer than REDIAL_EVICT_MIN_MS + * would evict the whole non-configured peerStore on the first dial after the + * connection came back. + */ +/** + * Whether zero-connection recovery may dial a DISCOVERED address this tick. + * + * It answers with the two records re-dial maintenance already keeps: a peer inside its + * backoff window waits for it to expire, and one still inside its unreachable + * quarantine stays down. Both are delays, never permanent bans — the point is only + * that the recovery loop must not undo the pacing the other loop just applied. + */ +export function isRecoveryDialDue(peerID: string, now: number, redialBackoff: ReadonlyMap, quarantine: ReadonlyMap): boolean { + const quarantinedAt = quarantine.get(peerID); + if (quarantinedAt !== undefined && now - quarantinedAt < UNREACHABLE_QUARANTINE_MS) return false; + const backoff = redialBackoff.get(peerID); + return backoff === undefined || backoff.nextAttempt <= now; +} + +export function nextEvictionWindowStart(reachable: boolean, previous: number | undefined, now: number): number { + return reachable ? (previous ?? now) : now; +} + +/** + * How many failures count TOWARDS EVICTION after another one. + * + * Counted separately from the backoff's failCount, which must keep growing through a + * local outage so we stop hammering the dialer. Eviction asks a different question — + * "has the PEER failed us N times?" — and a dial attempted while we had no connectivity + * answers nothing, so the run resets. Without this, two hours offline bank enough + * failures that the peer is evicted after the window even though only a couple of + * genuine failures happened once we were back: the backoff caps at 10 minutes, so a + * 30-minute window holds barely three attempts. + */ +export function nextEvictionFailCount(reachable: boolean, previous: number | undefined): number { + return reachable ? (previous ?? 0) + 1 : 0; +} + +/** + * Whether a run of re-dial failures has earned an eviction. + * + * Eviction is destructive — it purges the peerStore entry, drops the status row + * and quarantines the ID — so it needs all four conditions at once: we are + * demonstrably online, the peer has failed enough times, it has been failing for + * long enough, and it is not one the operator configured by hand. + */ +export function shouldEvictUnreachablePeer(input: { reachable: boolean; failCount: number; unreachableForMs: number; configured: boolean }): boolean { + if (!input.reachable || input.configured) return false; + return input.failCount >= REDIAL_EVICT_FAILS && input.unreachableForMs >= REDIAL_EVICT_MIN_MS; +} /** * Maximum size (bytes) of an incoming pubsub payload we are willing to decode. * Our own control messages ride pubsub (WANT — tiny JSON), but older/foreign peers @@ -102,11 +216,45 @@ const SEARCH_DEDUP_TTL_MS = 5 * 60_000; */ const MAX_PUBSUB_PAYLOAD_BYTES = 256 * 1024; +/** + * Where the node is in its start/stop cycle. + * + * `this.node` alone cannot express this: it is set half-way through {@link Network.start} + * and stays set for the whole of {@link Network.stop}, so a failed start left a node + * object that never started looking exactly like a running one, and a caller starting + * during a stop was told "already running" and then had its node torn down under it. + * Only a fully successful start reaches `running`. + * + * `failed` is the state of a stop that could not prove the node down. The run is neither + * running nor over: the node may still hold its listener, its connections and its port, + * and nothing has proved otherwise. Reporting `stopped` there is what allowed a second + * node over the same identity, port and datastore, and a datastore wipe underneath a live + * one. A `failed` whose libp2p stop was interrupted is permanent — libp2p cannot resume + * one, see {@link Network.teardown} — and only a process restart clears it. A `failed` + * whose libp2p stop succeeded and whose cleanup then failed can be left by a stop that + * completes the remaining cleanup. + */ +export type NetworkLifecycle = 'stopped' | 'starting' | 'running' | 'stopping' | 'failed'; + /** * Single shared libp2p node. * LISH networks are logical groups represented as pubsub topics on this one node. */ export class Network { + private lifecycle: NetworkLifecycle = 'stopped'; + /** + * Serialises start() against stop(). Both mutate the same fields across several + * awaits, and without this two concurrent start() calls could both pass the + * "already running" check before either created a node — two libp2p instances over + * one identity and one SQLite datastore. + */ + private readonly lifecycleMutex = new Mutex(); + /** + * Set once a `node.stop()` has failed to leave libp2p in `stopped`. Permanent: libp2p + * has no way to resume an interrupted stop, so every later attempt would be a no-op + * dressed up as success. See {@link teardown}. + */ + private nodeStopUnrecoverable = false; private node: Libp2p | null = null; private pubsub: PubSub | null = null; private datastore: SqliteDatastore | null = null; @@ -116,6 +264,23 @@ export class Network { private statusInterval: NodeJS.Timeout | null = null; /** Monotonic counter for status-interval ticks. Used by the periodic autodial promotion. */ private statusTickCount = 0; + /** + * Delayed peer-count probes armed by subscribeTopic. Tracked so stop() can cancel + * them: without that they keep a closure on this instance alive and can fire against + * a node the run no longer owns, which is exactly the ownership the epoch guards + * elsewhere are there to enforce. + */ + private readonly delayedPeerCountTimers: Set> = new Set(); + /** Guards against overlapping status ticks — see setupStatusInterval. */ + private statusTickInFlight = false; + /** + * Lifecycle epoch, bumped by stop(). A status tick captures the epoch at + * entry and refuses to write per-peer state once it differs — an in-flight + * tick otherwise survives stop() and would repopulate freshly-cleared maps + * or purge peers of the NEXT node instance (whose configured peers are not + * loaded yet) after a quick stop/start such as a factory reset. + */ + private runEpoch = 0; /** * Per-(peer,lish) timestamp of the last `have` response we sent. * Used to rate-limit responses to repeated `want` queries from the same peer for the same LISH: @@ -135,10 +300,35 @@ export class Network { * separate from bootstrapPeerIDs, which also collects peer-announce * discoveries: those are plain content peers and must remain * disconnectable by lishnet leave (isBootstrapOrRelayPeer). + * + * This is also what exempts a peer from unreachable-eviction: configured + * entries are user data, so a bootstrap hub that is down for half an hour must + * keep its peerStore entry and its addrs instead of being purged. Both + * questions — "is this infrastructure?" and "may we evict it?" — are the same + * question about the same fact, so they read the same set. Keeping two sets for + * it meant they could disagree, and they did: only one of them was ever pruned, + * so a peer the user had already removed from the config stayed eviction-exempt + * until restart. */ private configuredBootstrapPeerIDs: Set = new Set(); + /** + * Canonical bootstrap ADDRESSES that came from saved config, as opposed to gossip. + * + * Kept alongside the peer-ID set because the two answer different questions. Whether + * a PEER may be auto-evicted is about identity — configured anywhere means exempt. + * Whether an ADDRESS gets the configured treatment in recovery is about that address: + * one peer can have a configured address and a gossip-learned one at the same time, + * and the gossip-learned one must not inherit the exemption from its sibling. + */ + private readonly configuredBootstrapAddresses: Set = new Set(); private dcutrPeers: Set = new Set(); private bootstrapMultiaddrs: any[] = []; + /** + * Per-network bootstrap-config version, bumped on every replace / reset / leave. + * Read by {@link addBootstrapPeers} so a job started for a superseded list stops + * instead of re-adding entries that are no longer configured. + */ + private readonly bootstrapGeneration: Map = new Map(); // Topic handlers: topic -> Set of handler functions private topicHandlers: Map> = new Map(); @@ -211,8 +401,53 @@ export class Network { * capped at 10 min), so a persistently-unreachable peer does not saturate the * re-dial pool every 30s. Successful dial clears the entry. */ - private readonly redialBackoff = new Map(); - + private readonly redialBackoff = new Map(); + /** peerID → eviction time. Blocks re-adding a just-evicted unreachable peer from gossip for UNREACHABLE_QUARANTINE_MS. */ + private readonly unreachableQuarantine = new Map(); + /** + * Canonical multiaddr → pacing record for the loops that probe ONE CONFIGURED ADDRESS + * (zero-connection recovery and the parked-bootstrap probe). + * + * Keyed by ADDRESS, not by peer, because those loops ask an address-level question. + * One peer can hold a dead configured address and a working one at the same time; + * with a peer-keyed record the dead address's failure puts the whole peer into + * backoff, the working address is skipped for the rest of the pass, and the next pass + * starts at the dead one again — so the working address could go untried indefinitely. + */ + private readonly addressProbeBackoff = new Map(); + /** + * Canonical multiaddrs with an {@link addBootstrapPeers} dial currently outstanding. + * + * The pubsub dispatcher does not await the announce handler, so several announces + * naming the same address start several intake runs that overlap. Each spends its own + * 10 s dial timeout on the same endpoint and records its own outcome over the other's, + * and the peer-level backoff cannot help — it is only written once a dial has already + * failed. Claiming the address for the duration of the dial makes the second run a + * no-op instead: the first one is about to record the outcome both were after. + * + * Replaced with a FRESH Set on teardown rather than cleared, and captured by reference + * for the length of a dial. Clearing a shared Set let a dial still settling on the old + * node release a claim the new node's dial of the same address had just taken — after + * which a third request saw the address free and duplicated the live dial. + */ + private inFlightBootstrapDials = new Set(); + /** + * Peer IDs with a `peer:discovery` dial currently outstanding. + * + * Keyed by PEER, not by address, because that is the question discovery asks: mDNS, + * identify and PX all raise an event for the same arrival, each carrying its own + * address list, and the per-peer backoff cannot separate them — it is written only + * after a dial has already failed. Replaced on teardown for the same reason as + * {@link inFlightBootstrapDials}. + */ + private inFlightDiscoveryDials = new Set(); + /** + * peerID → time we first saw the peer disconnected with ZERO reachable + * addresses. Such peers never enter the re-dial path (nothing to dial), so + * the failure counter cannot evict them — without this they would sit in + * peerStore/bootstrap sets until maxPeerAge while every tick re-scans them. + */ + private readonly noReachableSince = new Map(); /** * Peers deliberately hung up by {@link disconnectPeer} (leave-network), keyed by * the lishnet they were left with. Redial maintenance / discovery must NOT @@ -246,10 +481,12 @@ export class Network { getNode: (): Libp2p | null => this.node, dialByPeerId: (peerID, protocol): Promise => this.dialProtocolByPeerId(peerID, protocol), }); + // Lets the discovered-row cap keep live participants and drop dead addresses first. + this.bootstrapTracker.setMembersProvider((networkID): Set => new Set(this.getTopicPeers(networkID))); this.peerAnnounce = new PeerAnnounceManager({ getNode: (): Libp2p | null => this.node, getPubsub: (): any => this.pubsub, - broadcast: (topic, msg): Promise => this.broadcast(topic, msg), + broadcast: (topic, msg, pubsub): Promise => Network.publishOn(pubsub, topic, msg), addBootstrapPeers: (multiaddrs, networkID, origin): Promise => this.addBootstrapPeers(multiaddrs, networkID, origin), }); } @@ -326,6 +563,17 @@ export class Network { /** * Schedule a debounced check of peer counts for all subscribed topics. */ + /** Arm a one-shot peer-count probe that stop() can still cancel. */ + private armDelayedPeerCountCheck(delayMs: number): void { + const epoch = this.runEpoch; + const timer = setTimeout(() => { + this.delayedPeerCountTimers.delete(timer); + if (epoch !== this.runEpoch) return; + this.schedulePeerCountCheck(); + }, delayMs); + this.delayedPeerCountTimers.add(timer); + } + private schedulePeerCountCheck(): void { if (this._peerCountDebounceTimer) clearTimeout(this._peerCountDebounceTimer); this._peerCountDebounceTimer = setTimeout(() => { @@ -378,11 +626,46 @@ export class Network { * @param bootstrapPeers - merged list of bootstrap peers from all enabled lishnets */ async start(bootstrapPeers: string[] = []): Promise { - if (this.node) { - console.log('Network node is already running'); - return; - } + // Serialised against stop() and against another start(): every field below is + // touched across awaits by both, so overlapping runs would interleave into two + // nodes over one datastore, or into a start whose node a concurrent stop tears + // down while its caller is told the start succeeded. + await this.lifecycleMutex.runExclusive(async () => { + // A previous run that could not be stopped may still own this identity, this + // port and this datastore. Starting a second node over it is the one thing + // that state exists to prevent, and "already running" would be a lie. + if (this.lifecycle === 'failed') throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network is in a failed state: the previous node could not be stopped'); + if (this.lifecycle !== 'stopped') { + console.log('Network node is already running'); + return; + } + this.lifecycle = 'starting'; + try { + await this.startLocked(bootstrapPeers); + this.lifecycle = 'running'; + } catch (err) { + // A half-built start owns a datastore handle and possibly a libp2p node. + // Leaving either behind is what made a failed start unrecoverable without + // restarting the process: the SQLite file stayed locked and `this.node` + // stayed set, so the next start reported "already running" forever. + try { + await this.teardown(); + } catch (teardownErr) { + // The cleanup could not prove the half-built node is down, so the next + // start must be refused rather than opening a second one over the same + // identity. Both reasons are kept: the start error explains what went + // wrong, the teardown error explains why the instance is now unusable. + this.lifecycle = 'failed'; + throw new AggregateError([err, teardownErr], 'network start failed and its cleanup could not complete'); + } + this.lifecycle = 'stopped'; + throw err; + } + }); + } + /** The body of {@link start}, run under the lifecycle mutex. */ + private async startLocked(bootstrapPeers: string[]): Promise { // Read settings const allSettings = this.settings.list(); @@ -547,29 +830,88 @@ export class Network { // re-dialed by discovery (mDNS/identify/PX) — that would beat the disconnect. // Suppression lifts on a legitimate inbound reconnect or on network rejoin. if (this.isRedialSuppressed(peerID)) return; - // Stamp `keep-alive-fleet` on every discovered peer, regardless of how they + // Stamp `keep-alive-fleet` on a peer we are actually CONNECTED to, however it // surfaced (mDNS, bootstrap, autonat, identify, peer-announce). libp2p // ReconnectQueue only acts on peers with a tag whose key starts with // `keep-alive`; without it, fleet peers found via non-announce channels // (e.g. identify push from a common neighbour) are not re-dialed when // they drop. Value 50 sits between bootstrap (100) and idle (1) — protects // from ConnectionPruner without taking precedence over true bootstraps. - try { - await this.node!.peerStore.merge(evt.detail.id, { - tags: { 'keep-alive-fleet': { value: 50 } }, - }); - } catch { - /* ignore */ - } + // + // Never on a mere discovery event, though: the tag is a standing instruction + // to libp2p to keep re-dialing this identity, and a discovery event is only + // somebody's claim that the peer exists. Stamping it before any contact let a + // peer that had just been evicted as unreachable get its re-dial instruction + // back from a late mDNS or PX event, ReconnectQueue included. + const tagAsFleetPeer = async (): Promise => { + try { + await this.node!.peerStore.merge(evt.detail.id, { + tags: { 'keep-alive-fleet': { value: 50 } }, + }); + } catch { + /* ignore */ + } + }; const existing = this.node!.getConnections(evt.detail.id); - if (existing.length > 0) return; + if (existing.length > 0) { + await tagAsFleetPeer(); + return; + } if (!evt.detail.multiaddrs?.length) return; + // The same pacing every other dial path respects. Discovery is a firehose — + // mDNS, identify and PX all deliver events for peers we have already written + // off — and this handler used to answer each one with an immediate dial, which + // could undo an eviction the moment it happened. + if (!isRecoveryDialDue(peerID, Date.now(), this.redialBackoff, this.unreachableQuarantine)) { + trace(`[NET] discovery dial skipped (quarantined or in backoff): ${peerID.slice(0, 16)}`); + return; + } + // Single-flight per peer. The backoff above is only written once a dial has + // already FAILED, so several discovery events for one peer — mDNS, identify and + // PX all fire for the same arrival — used to pass it together and start that + // many concurrent dials of the same identity. Captured by reference for the same + // reason the bootstrap claims are: a teardown replaces the set, and releasing + // into the replacement would free the next run's claim. + const inFlight = this.inFlightDiscoveryDials; + if (inFlight.has(peerID)) { + // Only the DIAL is skipped; the addresses are not lost. libp2p's own + // `#onDiscoveryPeer` merges every discovery service's address list into the + // peerStore before this public event is dispatched, so a second merge here + // would be an unguarded write that adds nothing — and one that can land after + // a leave, an eviction or a stop and reinstate addresses those just removed. + trace(`[NET] discovery dial skipped (already in flight): ${peerID.slice(0, 16)}`); + return; + } + inFlight.add(peerID); + const epoch = this.runEpoch; try { await this.node!.dial(evt.detail.multiaddrs); + // A dial that settles after stop() belongs to a node this run no longer owns. + if (epoch !== this.runEpoch) return; + // The suppression check at entry answered for the moment the event arrived, + // and a dial takes seconds. leave-network can land inside that window: its + // hangUp finds no connection yet, finishes, and this dial then completes into + // a connection nothing else will close. Whoever notices last closes it. + if (this.isRedialSuppressed(peerID) && !this.isPeerNeededByJoinedNetwork(peerID)) { + trace(`[NET] discovery dial landed after leave, hanging up: ${peerID.slice(0, 16)}`); + try { + await this.node!.hangUp(evt.detail.id); + } catch (err: any) { + trace(`[NET] hangUp of late discovery dial failed: ${err?.message ?? err}`); + } + return; + } + await tagAsFleetPeer(); trace(`[NET] Dialed discovered peer ${peerID.slice(0, 16)}`); } catch (err: any) { + if (epoch !== this.runEpoch) return; + // Pay the failure into the shared backoff the gate above reads, so a peer + // discovery keeps naming is paced like everything else. + this.noteRecoveryDialFailure(peerID); trace(`[NET] Failed to dial discovered peer ${peerID.slice(0, 16)}: ${err?.message ?? err}`); + } finally { + inFlight.delete(peerID); } }); @@ -577,6 +919,14 @@ export class Network { this.addListener(this.node!, 'peer:connect', async (evt: any) => { try { const peerID = evt.detail.toString(); + this.unreachableQuarantine.delete(peerID); + // Any verified connection resets the failure history — without this, a + // flappy NAT/relay peer that connects and drops BETWEEN status ticks + // keeps accumulating failCount across its live episodes and eventually + // gets evicted as "unreachable for 30 minutes" despite never being + // gone that long. + this.redialBackoff.delete(peerID); + this.noReachableSince.delete(peerID); const connections = this.node!.getConnections(evt.detail); const connTypes = connections.map(c => { const isRelay = Circuit.matches(c.remoteAddr); @@ -746,6 +1096,13 @@ export class Network { private setupStatusInterval(): void { this.statusInterval = setInterval(async () => { + // Serialize ticks: with many unreachable peers the re-dial phase (5 s + // timeout × candidates ÷ concurrency) can exceed the 30 s cadence. Two + // interleaved ticks would race on redialBackoff — one tick could evict + // (and close connections of) a peer another tick just reconnected. + if (this.statusTickInFlight) return; + this.statusTickInFlight = true; + const epoch = this.runEpoch; try { const connectedPeers = this.node!.getPeers(); const allPeers = await this.node!.peerStore.all(); @@ -753,11 +1110,33 @@ export class Network { dumpGossipsubScores({ node: this.node, pubsub: this.pubsub, settings: this.settings, lastScores: this._lastScores }, connectedPeers); // Periodic peer count refresh — catches cases where GRAFT/PRUNE events were missed this.checkPeerCounts(); - await this.runRedialMaintenance(connectedPeers, allPeers); - await this.runZeroConnectionRecovery(connectedPeers); - await this.maybePromotePeers(); + await this.runRedialMaintenance(connectedPeers, allPeers, epoch); + if (epoch !== this.runEpoch) return; + await this.runZeroConnectionRecovery(epoch); + if (epoch !== this.runEpoch) return; + await this.maybePromotePeers(epoch); + if (epoch !== this.runEpoch) return; + // Sweep by per-network membership (topic subscribers), not global + // connectivity: a peer that left this network but stays connected via + // another must still have its stale row here expire. Snapshot per topic + // lazily and freshly — the tick-start state is stale after the re-dial + // phase, and a peer that (re)subscribed during it must not be swept. + const topicMembers = new Map>(); + const isMember = (networkID: string, pid: string): boolean => { + let set = topicMembers.get(networkID); + if (!set) { + set = new Set(this.getTopicPeers(networkID)); + topicMembers.set(networkID, set); + } + return set.has(pid); + }; + this.bootstrapTracker.sweepStale(BOOTSTRAP_STATUS_STALE_MS, isMember); } catch (err: any) { trace(`[NET] statusInterval error: ${err?.message ?? err}`); + } finally { + // Only release the guard for the run we belong to — after a stop()/start() + // the flag belongs to the new run, whose own tick may already hold it. + if (epoch === this.runEpoch) this.statusTickInFlight = false; } }, 30000); // Status interval 30 s. promoteKnownPeersToBootstrap + gossipsub.direct @@ -799,7 +1178,7 @@ export class Network { for (const set of this.redialSuppressedByNet.values()) set.delete(peerID); } - private async runRedialMaintenance(connectedPeers: any[], allPeers: any[]): Promise { + private async runRedialMaintenance(connectedPeers: any[], allPeers: any[], epoch: number = this.runEpoch): Promise { // Dial known peers not currently connected (maintains relay connections to NATed peers) const connectedSet = new Set(connectedPeers.map(p => p.toString())); const now = Date.now(); @@ -810,11 +1189,15 @@ export class Network { let skippedBackoff = 0; let skippedNoReachable = 0; let skippedSuppressed = 0; + let skippedQuarantined = 0; const localCidrs = getLocalCidrs(now); for (const peer of allPeers) { + if (epoch !== this.runEpoch) return; // stop() hit — this run's state is gone const pid = peer.id.toString(); if (connectedSet.has(pid)) { this.redialBackoff.delete(pid); // clear on observed connection + this.unreachableQuarantine.delete(pid); + this.noReachableSince.delete(pid); if (this.sharesJoinedTopicWith(pid)) this.clearRedialSuppressionForPeer(pid); // back on a shared topic → resume continue; } @@ -824,6 +1207,17 @@ export class Network { skippedSuppressed++; continue; } + // A peer evicted as unreachable is normally gone from the peerStore and so + // cannot be a candidate at all — but that delete is best-effort, and mDNS, + // identify and peer-announce can all put the entry back. Honour the quarantine + // here too, or the very next tick dials the peer we just wrote off. Configured + // peers are never quarantined; the check is stated anyway so no future writer + // can hold user data back by adding one. + const quarantinedAt = this.unreachableQuarantine.get(pid); + if (quarantinedAt !== undefined && now - quarantinedAt < UNREACHABLE_QUARANTINE_MS && !this.configuredBootstrapPeerIDs.has(pid)) { + skippedQuarantined++; + continue; + } const bo = this.redialBackoff.get(pid); if (bo && bo.nextAttempt > now) { skippedBackoff++; @@ -842,8 +1236,31 @@ export class Network { } if (reachable.length === 0) { skippedNoReachable++; + // No dialable address ⇒ the failure counter can never fire for this + // peer. Track how long it has been in this state; a disconnected peer + // with zero reachable addrs for the whole eviction window is as gone + // as one that failed every dial. + // + // "Unreachable" here means the dial gater rejects every stored address + // from where WE stand, which is not the same as the peer being gone: a + // peer reachable only over a LAN or VPN subnet stops passing the filter + // the moment that interface drops, through no fault of its own. So this + // path takes the same two safeguards as the dial-failure path below — + // evidence that we are online at all, and a liveness re-check right + // before acting — on top of the configured-peer exemption. + const weAreOnline = this.hasConnectionOtherThan(peer.id); + const since = nextEvictionWindowStart(weAreOnline, this.noReachableSince.get(pid), now); + this.noReachableSince.set(pid, since); + if (weAreOnline && now - since >= REDIAL_EVICT_MIN_MS && !this.configuredBootstrapPeerIDs.has(pid) && this.node?.getConnections(peer.id).length === 0) { + this.noReachableSince.delete(pid); + this.unreachableQuarantine.set(pid, now); + this.redialBackoff.delete(pid); + this.bootstrapTracker.deleteDiscoveredByPeerID(pid); + await this.purgeStalePeer(pid, `no reachable addresses for ${Math.round((now - since) / 60_000)} min`, epoch); + } continue; } + this.noReachableSince.delete(pid); // addresses came back — reset the clock candidates.push({ peer, pid, addrSummary: reachable.join(' | '), failCount: bo?.failCount ?? 0 }); } // Parallel dial with concurrency=10 via rolling promise pool; caps worst-case @@ -853,10 +1270,14 @@ export class Network { let idx = 0; const worker = async (): Promise => { while (idx < candidates.length) { + if (epoch !== this.runEpoch) return; // stop() hit — abandon remaining dials const c = candidates[idx++]!; console.debug(` ↻ Re-dial attempt peer=${c.pid} addrs=${c.addrSummary} fails=${c.failCount}`); try { await this.node!.dial(c.peer.id, { signal: AbortSignal.timeout(5000) }); + // Same guard as the failure path: a dial resolving after stop() must + // not write into the next run's state or the next node's peerStore. + if (epoch !== this.runEpoch) return; const conns = this.node!.getConnections(c.peer.id); const connDetail = conns .map(conn => { @@ -880,18 +1301,47 @@ export class Network { /* non-fatal */ } } catch (err: any) { + // A dial aborted by stop() looks like any other failure — do not let + // it repopulate maps that stop() just cleared, or evict against the + // NEXT node instance. + if (epoch !== this.runEpoch) return; // Exponential backoff: 30s × 2^failCount, capped at 10 min. const nextFailCount = c.failCount + 1; const delayMs = Math.min(30_000 * 2 ** c.failCount, 600_000); - this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount }); + const reachable = this.hasConnectionOtherThan(c.peer.id); + const previous = this.redialBackoff.get(c.pid); + const firstFailure = nextEvictionWindowStart(reachable, previous?.firstFailure, Date.now()); + const evictionFails = nextEvictionFailCount(reachable, previous?.evictionFails); + this.redialBackoff.set(c.pid, { nextAttempt: Date.now() + delayMs, failCount: nextFailCount, firstFailure, evictionFails }); console.debug(` ✗ Re-dial peer=${c.pid} failed: ${err.message ?? err} (tried: ${c.addrSummary}, next in ${Math.round(delayMs / 1000)}s)`); + // Enough consecutive failures over enough time ⇒ the peer is gone, not + // flaky. The dial above went by peer ID, so libp2p tried EVERY known + // address — one broken addr among working ones cannot trip this. Evict + // everywhere (peerStore, bootstrap sets, discovered status rows) and + // quarantine the ID so gossip mentions don't immediately re-add it. + // Configured bootstrap peers are exempt — user data, they must survive + // any outage and keep their red status row instead. + if (shouldEvictUnreachablePeer({ reachable, failCount: evictionFails, unreachableForMs: Date.now() - firstFailure, configured: this.configuredBootstrapPeerIDs.has(c.pid) })) { + // Last-moment liveness check: the peer may have connected (inbound + // dial, another async path) while this worker was failing on stale + // state. purgeStalePeer closes connections, so evicting here would + // cut a LIVE peer — verify emptiness right before acting. + if (this.node && this.node.getConnections(c.peer.id).length > 0) { + this.redialBackoff.delete(c.pid); + continue; + } + this.unreachableQuarantine.set(c.pid, Date.now()); + this.redialBackoff.delete(c.pid); + this.bootstrapTracker.deleteDiscoveredByPeerID(c.pid); + await this.purgeStalePeer(c.pid, `unreachable after ${evictionFails} re-dial failures over ${Math.round((Date.now() - firstFailure) / 60_000)} min`, epoch); + } } } }; const workers = Array.from({ length: Math.min(CONCURRENCY, candidates.length) }, () => worker()); await Promise.all(workers); - if (candidates.length > 0 || skippedBackoff > 0 || skippedNoReachable > 0 || skippedSuppressed > 0) { - console.debug(` Re-dial: ${redialSuccess}/${candidates.length} succeeded (${skippedBackoff} skipped by backoff, ${skippedNoReachable} skipped no-reachable-addrs, ${skippedSuppressed} skipped left-peer)`); + if (candidates.length > 0 || skippedBackoff > 0 || skippedNoReachable > 0 || skippedSuppressed > 0 || skippedQuarantined > 0) { + console.debug(` Re-dial: ${redialSuccess}/${candidates.length} succeeded (${skippedBackoff} skipped by backoff, ${skippedNoReachable} skipped no-reachable-addrs, ${skippedSuppressed} skipped left-peer, ${skippedQuarantined} skipped quarantined)`); } // Prune backoff entries for peers no longer in peerStore to prevent unbounded growth. // Suppression is NOT pruned this way: leave-network purges the peer from the @@ -900,10 +1350,89 @@ export class Network { // bounded by clear-on-rejoin / clear-on-reconnect / stop(). const storeSet = new Set(allPeers.map(p => p.id.toString())); for (const pid of this.redialBackoff.keys()) if (!storeSet.has(pid)) this.redialBackoff.delete(pid); + for (const pid of this.noReachableSince.keys()) if (!storeSet.has(pid)) this.noReachableSince.delete(pid); + // Quarantine entries for peers gossip never mentions again would leak — drop + // them once they are far past the window (re-entry from gossip self-cleans). + const quarantineCutoff = now - 2 * UNREACHABLE_QUARANTINE_MS; + for (const [pid, ts] of this.unreachableQuarantine) if (ts < quarantineCutoff) this.unreachableQuarantine.delete(pid); } - private async runZeroConnectionRecovery(connectedPeers: any[]): Promise { - if (!AUTODIAL_WORKAROUND || connectedPeers.length !== 0 || this.bootstrapMultiaddrs.length === 0) return; + /** + * Whether a per-ADDRESS probe of a configured entry is due. + * + * An address nothing has failed on yet is always due; one that failed waits out the + * window {@link noteAddressProbeFailure} set for it. + */ + private isAddressProbeDue(canonicalAddress: string, now: number): boolean { + const entry = this.addressProbeBackoff.get(canonicalAddress); + return entry === undefined || entry.nextAttempt <= now; + } + + /** + * Record a failed probe of a configured ADDRESS: 30 s × 2^fails, capped at + * {@link CONFIGURED_PROBE_BACKOFF_MAX_MS}. + * + * Paces and nothing more — a configured entry is exempt from eviction and from + * quarantine, so this record must never become the evidence that removes one. + */ + private noteAddressProbeFailure(canonicalAddress: string): void { + const failCount = (this.addressProbeBackoff.get(canonicalAddress)?.failCount ?? 0) + 1; + this.addressProbeBackoff.set(canonicalAddress, { nextAttempt: Date.now() + Math.min(30_000 * 2 ** (failCount - 1), CONFIGURED_PROBE_BACKOFF_MAX_MS), failCount }); + } + + /** + * Record a failed recovery dial of a DISCOVERED address against the shared per-peer + * backoff, in the same four-field shape every other writer uses. + * + * Without this the recovery loop was the one dial path that paced nothing: for an + * address whose peer is not in the peerStore, re-dial maintenance never sees the peer + * either, so a dead entry was re-dialed on every tick for as long as the node stayed + * isolated. `evictionFails` deliberately does not grow — at zero connections there is + * no evidence the remote is the broken side, which is exactly what + * {@link nextEvictionFailCount} resets on. + */ + private noteRecoveryDialFailure(peerID: string): void { + const now = Date.now(); + const previous = this.redialBackoff.get(peerID); + const failCount = previous?.failCount ?? 0; + this.redialBackoff.set(peerID, { + nextAttempt: now + Math.min(30_000 * 2 ** failCount, 600_000), + failCount: failCount + 1, + firstFailure: previous?.firstFailure ?? now, + evictionFails: previous?.evictionFails ?? 0, + }); + // An expired quarantine is what let this dial through, and it buys exactly one + // probe: re-arm it on failure or every later pass spends another dial on a peer + // that has already been written off once. + if (this.unreachableQuarantine.has(peerID)) this.unreachableQuarantine.set(peerID, now); + } + + /** + * Whether some open connection already terminates on the exact endpoint an address + * names. + * + * The peer-level question — "are we connected to them at all" — is the wrong one for a + * per-address probe: a peer reachable through a second address would mask a broken + * configured entry for as long as that other route held, which is precisely the case + * the probe exists to expose. + */ + private hasConnectionOnEndpoint(ma: any): boolean { + if (!this.node) return false; + const target = ma.toString(); + try { + return this.node.getConnections().some(c => isSameDialEndpoint(String(c.remoteAddr ?? ''), target)); + } catch { + return false; + } + } + + private async runZeroConnectionRecovery(epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; + // Read connectivity here rather than trusting the snapshot the status tick opened + // with: re-dial maintenance runs in between and may already have reconnected us, + // in which case the node is not isolated and every dial below is pure churn. + if (!AUTODIAL_WORKAROUND || node.getPeers().length !== 0 || this.bootstrapMultiaddrs.length === 0) return; console.log(` ⚠️ No connections - dialing ${this.bootstrapMultiaddrs.length} bootstrap peer(s) directly...`); // [NET-CHURN] dump: who left in the run-up to this zero-connection // state, and what each configured bootstrap entry's last dial outcome @@ -924,31 +1453,121 @@ export class Network { console.log(` [NET-CHURN] bootstrap stats net=${networkID.slice(0, 8)}: ${parts}`); } for (const ma of this.bootstrapMultiaddrs) { - const p2pComponents = ma.getComponents().filter((c: { code: number; value?: string }) => c.code === 421); - const pid: string | undefined = p2pComponents.length > 0 ? p2pComponents[p2pComponents.length - 1].value : undefined; + const pid = extractDestinationPeerID(ma); if (pid && this.isRedialSuppressed(pid)) continue; // deliberately left — don't resurrect it here + // A CONFIGURED entry is the user's way back in, so it is never held back by the + // quarantine or by the per-peer eviction backoff — but it is still paced, on its + // own much shorter ADDRESS-level window. Without any pacing, several dead + // configured entries at a 10 s timeout each turn one tick into minutes of + // back-to-back dialing, every tick. + // + // A DISCOVERED entry earned its place here by answering once, which is no reason + // to bypass the pacing re-dial maintenance applies to it. Without that, an + // isolated node re-dialed a dead discovered peer every 30 s forever, since + // maintenance stops counting failures the moment we have no other connection to + // prove we are online. + const canonical = normalizeMultiaddrForCompare(ma.toString()); + const configured = this.configuredBootstrapAddresses.has(canonical); + if (configured) { + if (!this.isAddressProbeDue(canonical, Date.now())) continue; + } else if (pid && !isRecoveryDialDue(pid, Date.now(), this.redialBackoff, this.unreachableQuarantine)) { + continue; + } + // Routability is re-checked here, not just at configure time: a LAN or VPN + // bootstrap is on this list while its interface is down, and becomes dialable + // again the moment it returns. + if (shouldDenyDial(ma, getLocalCidrs())) continue; const maStr = ma?.toString?.() ?? String(ma); + // Each dial awaits for up to 10s, so a stop() can land mid-loop; the + // remaining dials belong to a node this run no longer owns. + if (epoch !== this.runEpoch) return; + // The whole point of this loop is isolation. A dial from an earlier pass that + // resolved late, or an inbound connection, ends it — carrying on would open + // connections the node no longer needs. + if (node.getPeers().length > 0) return; try { console.log(` → Dialing ${maStr}`); - await this.node!.dial(ma, { signal: AbortSignal.timeout(10000) }); + await node.dial(ma, { signal: AbortSignal.timeout(10000) }); + if (epoch !== this.runEpoch) return; + if (configured) this.addressProbeBackoff.delete(canonical); + else if (pid) this.redialBackoff.delete(pid); console.log(` ✓ Connected via ${maStr}`); break; } catch (err: any) { + if (epoch !== this.runEpoch) return; + if (configured) this.noteAddressProbeFailure(canonical); + else if (pid) this.noteRecoveryDialFailure(pid); console.log(` ✗ Failed ${maStr}: ${err.message ?? err}`); } } } - private async maybePromotePeers(): Promise { + /** + * Slowly re-probe CONFIGURED bootstrap addresses that nothing else will reach. + * + * An address the routability filter rejected at configure time — a LAN or VPN + * bootstrap whose interface was down — never entered the peerStore, so re-dial + * maintenance (which walks the peerStore) has no candidate for it. Zero-connection + * recovery would pick it up, but only while the node has NO connections at all, so a + * node happily talking to someone else would never notice the tunnel came back. + * + * Runs on the slow promote cadence and paces itself per ADDRESS, so a permanently + * broken entry costs one dial per window and cannot starve a sibling address of the + * same peer that does work. + */ + private async probeParkedConfiguredBootstraps(epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; + const localCidrs = getLocalCidrs(); + for (const ma of [...this.bootstrapMultiaddrs]) { + if (epoch !== this.runEpoch) return; + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (!this.configuredBootstrapAddresses.has(canonical)) continue; + const pid = extractDestinationPeerID(ma); + if (pid && this.isRedialSuppressed(pid)) continue; + // Still unreachable from here — leave it parked for a later pass. + if (shouldDenyDial(ma, localCidrs)) continue; + if (!this.isAddressProbeDue(canonical, Date.now())) continue; + // Only a connection ON THIS ENDPOINT answers the question the probe asks. A + // connection to the same peer over some other address used to skip it, which + // is exactly how a broken configured entry kept looking fine. + if (this.hasConnectionOnEndpoint(ma)) continue; + try { + // Forced for the same reason the configured branch of addBootstrapPeers + // forces: without it libp2p hands back whatever connection it already holds + // to this peer and the probe proves nothing about the address. + await node.dial(ma, { signal: AbortSignal.timeout(10000), force: true }); + if (epoch !== this.runEpoch) return; + this.addressProbeBackoff.delete(canonical); + // Tell the UI as well. This probe is the ONLY thing that retries an address + // the routability filter rejected at configure time, so without this the row + // written when the interface was down stayed red for as long as the node ran, + // however long the address had since been working. + this.bootstrapTracker.recordAddressReachable(ma.toString()); + console.log(`[NET] parked configured bootstrap reachable again: ${ma.toString()}`); + } catch (err: any) { + if (epoch !== this.runEpoch) return; + this.noteAddressProbeFailure(canonical); + trace(`[NET] parked configured bootstrap still failing: ${ma.toString()} — ${err?.message ?? err}`); + } + } + } + + private async maybePromotePeers(epoch: number = this.runEpoch): Promise { // Every 5th status tick (~150 s at 30 s status cadence) promote every - // peerStore entry back to bootstrap priority. Re-stamps KEEP_ALIVE tags - // and feeds libp2p a concrete multiaddr list to re-dial against, catching - // peers whose original dial cached a stale (unreachable) address — these - // would otherwise sit idle until they reappeared via identify/PX/announce. + // CONNECTED peer back to bootstrap priority (KEEP_ALIVE re-stamp + gossipsub + // direct set). Disconnected peers are handled by runRedialMaintenance. this.statusTickCount++; if (this.statusTickCount % 5 === 0) { try { - await this.promoteKnownPeersToBootstrap(); + // Same slow cadence: an address parked as unroutable has no other loop + // that would ever notice its interface came back. + await this.probeParkedConfiguredBootstraps(epoch); + } catch (err: any) { + trace(`[NET] probeParkedConfiguredBootstraps failed: ${err?.message ?? err}`); + } + try { + await this.promoteKnownPeersToBootstrap(epoch); } catch (err: any) { trace(`[NET] promoteKnownPeersToBootstrap failed: ${err?.message ?? err}`); } @@ -956,52 +1575,104 @@ export class Network { } /** - * Promote every known peer (from libp2p peerStore) back to bootstrap priority so - * KEEP_ALIVE tagging and direct-dial re-runs cover peers the ordinary re-dial loop - * skipped because their cached multiaddrs looked like loopback/private-IP garbage. - * Runs every ~45 s from the status tick. + * Promote every CONNECTED peer back to bootstrap priority: KEEP_ALIVE tagging, + * bootstrap dedup-set membership, and gossipsub direct-set fast reconnect. + * Disconnected peers are deliberately excluded — runRedialMaintenance already + * dials each of them every tick with exponential backoff and eviction, whereas + * promotion dials have no backoff, so including them meant a burst of dials to + * dead peers every promotion cycle and their permanent growth in the direct set. + * Runs every ~150 s from the status tick. */ - private async promoteKnownPeersToBootstrap(): Promise { + private async promoteKnownPeersToBootstrap(epoch: number = this.runEpoch): Promise { if (!this.node) return; const allPeers = await this.node.peerStore.all(); + // stop() may have landed while peerStore.all() was pending — promoting now + // would repopulate bootstrap/tracker state the shutdown just cleared (or, + // after a fast restart, populate the NEXT node from the old snapshot). + if (epoch !== this.runEpoch) return; const myID = this.node.peerId.toString(); + const connectedIDs = new Set(this.node.getPeers().map((p: any) => p.toString())); const maStrings: string[] = []; for (const peer of allPeers) { const pid = peer.id.toString(); if (pid === myID) continue; + if (!connectedIDs.has(pid)) continue; if (this.isRedialSuppressed(pid)) continue; // deliberately left — don't promote it back to bootstrap if (this.bootstrapPeerIDs.has(pid)) continue; if (peer.addresses.length === 0) continue; const addr = peer.addresses[0]!; const base = addr.multiaddr.toString(); - // Ensure /p2p/ suffix — addBootstrapPeers extracts peer ID via multiaddr component 421. - const maStr = base.includes('/p2p/') ? base : `${base}/p2p/${pid}`; + // Ensure the address terminates in THIS peer's /p2p/ — a bare address + // gets the suffix appended, and so does a relay address whose only /p2p/ + // component is the relay's own identity. + const maStr = extractDestinationPeerID(addr.multiaddr) === pid ? base : `${base}/p2p/${pid}`; maStrings.push(maStr); } - if (maStrings.length === 0) return; - trace(`[NET] periodic autodial: promoting ${maStrings.length} peer(s) to bootstrap`); - await this.addBootstrapPeers(maStrings); - // Also insert every known peer into the gossipsub `direct` Set at runtime. - // Direct peers are never PRUNED by D/Dhi and have their own fast reconnect - // cadence (directConnectTicks × heartbeatInterval). KEEP_ALIVE handles the - // TCP layer; gossipsub.direct handles the gossipsub-stream layer. + if (maStrings.length > 0) { + trace(`[NET] periodic autodial: promoting ${maStrings.length} connected peer(s) to bootstrap`); + await this.addBootstrapPeers(maStrings); + if (epoch !== this.runEpoch) return; + } + // Also insert every connected peer into the gossipsub `direct` Set at runtime. + // Direct peers have their own fast reconnect cadence (directConnectTicks × + // heartbeatInterval). KEEP_ALIVE handles the TCP layer; gossipsub.direct + // handles the gossipsub-stream layer. Evicted peers are removed from the + // set in purgeStalePeer, so it no longer grows monotonically. + let added = 0; + for (const peer of allPeers) { + const pid = peer.id.toString(); + if (pid === myID) continue; + if (!connectedIDs.has(pid)) continue; + if (this.addGossipsubDirectPeer(pid)) added++; + } + if (added > 0) trace(`[NET] gossipsub direct: added ${added} connected peer(s) to fast-reconnect set`); + } + + /** + * Put a peer into the gossipsub `direct` set — never PRUNEd, and reconnected on its + * own fast cadence (directConnectTicks × heartbeatInterval). Removed again by + * {@link purgeStalePeer}. Returns whether this call was the one that added it. + * + * A left-network peer that lingers or reappears in the peerStore is refused: the fast + * reconnect cadence would undo the leave-network disconnect. + */ + private addGossipsubDirectPeer(peerID: string): boolean { const gossipsub: any = this.pubsub; - if (gossipsub?.direct && typeof gossipsub.direct.add === 'function') { - let added = 0; - for (const peer of allPeers) { - const pid = peer.id.toString(); - if (pid === myID) continue; - // A left-network peer that lingers/reappears in the peerStore must not be - // added to the direct set either — its fast reconnect cadence would undo the - // leave-network disconnect (same guard as the bootstrap promotion above). - if (this.isRedialSuppressed(pid)) continue; - if (!gossipsub.direct.has(pid)) { - gossipsub.direct.add(pid); - added++; - } + if (!gossipsub?.direct || typeof gossipsub.direct.add !== 'function') return false; + if (this.isRedialSuppressed(peerID)) return false; + if (gossipsub.direct.has(peerID)) return false; + gossipsub.direct.add(peerID); + return true; + } + + /** Undo {@link addGossipsubDirectPeer}: no more never-PRUNE, no more fast redial. */ + private removeGossipsubDirectPeer(peerID: string): void { + const gossipsub: any = this.pubsub; + if (gossipsub?.direct && typeof gossipsub.direct.delete === 'function') gossipsub.direct.delete(peerID); + } + + /** + * Drop the KEEP_ALIVE tag a bootstrap entry earned by answering a dial. + * + * libp2p re-dials anything carrying a keep-alive tag and the connection manager will + * not evict it, so a configured entry the user has deleted stays pinned for the life + * of the run unless the tag goes with it. Kept if a joined network still wants the + * peer — the tag is then no longer the bootstrap lifecycle's to take away. + * + * The merge is bound to the node captured here rather than `this.node`, so a restart + * landing in it cannot untag the same identity on the successor run. + */ + private clearBootstrapKeepAlive(peerID: string): void { + const node = this.node; + if (!node) return; + if (this.isPeerNeededByJoinedNetwork(peerID)) return; + void (async (): Promise => { + try { + await node.peerStore.merge(peerIDFromString(peerID), { tags: { [KEEP_ALIVE]: undefined } }); + } catch (err: any) { + trace(`[NET] clearBootstrapKeepAlive failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } - if (added > 0) trace(`[NET] gossipsub direct: added ${added} known peer(s) to never-PRUNE set`); - } + })(); } // ========================================================================= @@ -1012,7 +1683,12 @@ export class Network { * Whether the node is running. */ isRunning(): boolean { - return this.node !== null; + return this.lifecycle === 'running'; + } + + /** Current start/stop phase. Exposed for tests and diagnostics. */ + getLifecycle(): NetworkLifecycle { + return this.lifecycle; } /** @@ -1034,32 +1710,70 @@ export class Network { * additions with no owning network, in which case stats are skipped. */ async addBootstrapPeers(peers: string[], networkID: string | null = null, origin: BootstrapPeerOrigin = 'discovered'): Promise { + // Group the run's status writes. Intake performs two per address — a pending mark + // and an outcome — and each rebuilds and publishes the network's whole peer list, + // so one 128-address announce used to cost 256 snapshots and 256 WebSocket pushes + // of which the UI kept the last. The frame flushes periodically rather than only + // at close, so a list of slow dials still reports progress as it goes. + if (networkID === null) return this.dialBootstrapEntries(peers, networkID, origin); + await this.bootstrapTracker.batchDebounced(networkID, () => this.dialBootstrapEntries(peers, networkID, origin)); + } + + /** The dial loop behind {@link addBootstrapPeers}; see there for the batching wrapper. */ + private async dialBootstrapEntries(peers: string[], networkID: string | null, origin: BootstrapPeerOrigin): Promise { if (!this.node) { console.error('Network not started - cannot add bootstrap peers'); return; } const myPeerID = this.node.peerId.toString(); const localCidrs = getLocalCidrs(); + // Fire-and-forget callers (peer-announce intake, startup joins) run outside + // the status-tick epoch guard. Capture the epoch so a dial that settles after + // a stop()/restart cannot record outcomes on the cleared tracker or write + // peerStore state for the NEXT node instance. + // + // The generation covers the other axis: the node stays up but THIS network's + // bootstrap list is replaced, reset or left while we are part-way down it. The + // loop dials sequentially and one dial can take seconds, so an old job would + // otherwise keep walking the old list — re-adding entries the user has just + // removed and re-marking them configured, which exempts them from the stale + // sweep until restart. Exactly the resurrection this eviction work exists to + // prevent. + const epoch = this.runEpoch; + // Captured, not re-read: the claims this run takes have to be released back into + // the same object it took them from, whatever teardown has since put in the field. + const inFlight = this.inFlightBootstrapDials; + const generation = this.bootstrapGenerationOf(networkID); + const superseded = (): boolean => epoch !== this.runEpoch || generation !== this.bootstrapGenerationOf(networkID); for (const peer of peers) { - // Skip our own address - if (peer.includes(myPeerID)) continue; + if (superseded()) return; + let probeAfterQuarantine = false; try { const ma = Multiaddr(peer); - // Safety net: refuse to add loopback / unreachable-private bootstrap - // entries even if the upstream (catalog or peer-announce intake) - // failed to filter them. Failing here is silent because the call - // site iterates many candidates and we shouldn't spam INFO for - // every drop; trace-level keeps it greppable when debugging. - if (shouldDenyDial(ma, localCidrs)) { - trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); - continue; - } - // A relayed bootstrap multiaddr (.../p2p//p2p-circuit/p2p/) - // carries two /p2p components; the peer we actually connect to — and must - // exempt from leave-network disconnect — is the FINAL one (the target), not - // the relay. Take the last /p2p component, never the first. - const p2pComponents = ma.getComponents().filter(c => c.code === 421); - const peerID = (p2pComponents.length > 0 ? p2pComponents[p2pComponents.length - 1]!.value : null) ?? null; + // Claim the configured status BEFORE the routability filter. Whether an address + // is dialable is a property of THIS HOST right now — a LAN or VPN bootstrap stops + // passing the filter the moment that interface drops — while "the user configured + // this peer" is a fact about the saved config. Deriving the second from the first + // left a VPN bootstrap unregistered whenever the tunnel was down at startup, so the + // exemption that makes configured peers un-evictable never applied to it. + const peerID = extractDestinationPeerID(ma); + // Skip our own address — compare the DESTINATION identity, not the raw + // string: `/p2p//p2p-circuit/p2p/` contains our ID as the + // relay hop yet targets a remote peer and must not be dropped as self. + if (peerID === myPeerID) continue; + // What this address IS outranks what this caller calls it. The status tracker + // already keeps the stronger classification when a row is overwritten, so a + // gossip re-announcement of an address the user configured lands on a + // CONFIGURED row — and the dial has to follow the same rule or the two + // disagree. They did: the announce dialed without `force`, libp2p handed back + // the connection the peer happened to hold on a DIFFERENT address, and the + // discovered branch recorded 'connected' on a configured row whose address had + // never been contacted. The user then saw a green light on a broken entry. + const canonicalAddress = normalizeMultiaddrForCompare(ma.toString()); + const effectiveOrigin: BootstrapPeerOrigin = origin === 'configured' || this.configuredBootstrapAddresses.has(canonicalAddress) ? 'configured' : 'discovered'; + // Claiming the peer as configured stays keyed on what the CALLER declared: + // this branch also lifts leave-network suppression, which is the user's + // decision to reverse, never gossip's. if (peerID && origin === 'configured') { this.configuredBootstrapPeerIDs.add(peerID); // A re-configured bootstrap peer means its network was (re-)joined — it @@ -1068,59 +1782,258 @@ export class Network { // explicit dial fails or the connection drops before the next tick. this.clearRedialSuppressionForPeer(peerID); } - const alreadyKnown = !!peerID && this.bootstrapPeerIDs.has(peerID); - if (peerID && !alreadyKnown) { - this.bootstrapPeerIDs.add(peerID); - this.bootstrapMultiaddrs.push(ma); + // Also before the routability filter: a LAN or VPN bootstrap is unroutable only + // while its interface is down, and keeping it off the recovery list until then + // means nothing retries it when the tunnel returns. Recovery re-checks + // routability itself before dialing. + // The autodial list is a different promise: zero-connection recovery walks + // it and dials everything on it. A CONFIGURED address belongs there at once + // — it is user data and recovery must keep trying it precisely while it is + // down. A DISCOVERED address is only a claim some peer made, so it earns + // its place by answering; it is added after a verified dial, below. Adding + // it here left every unreachable address a gossip flood could invent on the + // list for good, since an ordinary timeout has nothing that takes it off. + if (origin === 'configured') { + this.configuredBootstrapAddresses.add(canonicalAddress); + this.rememberBootstrapAddress(ma); } + // Safety net: refuse to dial loopback / unreachable-private bootstrap entries + // even if the upstream (catalog or peer-announce intake) failed to filter them. + // A discovered address is dropped silently — the call site iterates many + // candidates and should not spam INFO for each. A configured one gets a status + // row instead: the user wrote it down and needs to see why nothing happens with it. + if (shouldDenyDial(ma, localCidrs)) { + trace(`[NET] addBootstrapPeers skip non-routable: ${peer}`); + if (effectiveOrigin === 'configured') this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'error', 'address is not routable from this host', null, effectiveOrigin); + continue; + } + // Pacing, before the quarantine is consulted: gossip mentions a dead peer + // on every announce cycle, and this path used to answer each one with a + // fresh 10 s dial because it read the quarantine and nothing else. The + // backoff is the record every other dial path already waits on, and + // checking it FIRST matters — an expired quarantine buys exactly one probe, + // which must not be spent by a dial the backoff was going to refuse. + if (peerID && effectiveOrigin === 'discovered') { + const backoff = this.redialBackoff.get(peerID); + if (backoff !== undefined && backoff.nextAttempt > Date.now()) { + trace(`[NET] addBootstrapPeers skip backoff: ${peerID.slice(0, 16)}`); + continue; + } + } + // Skip peers recently evicted as unreachable — nodes that still remember + // them keep gossiping their addrs, and without this window every mention + // would re-create the status row and burn a dial. Configured entries are + // exempt: the user asked for them explicitly. + if (peerID && effectiveOrigin === 'discovered') { + const quarantinedAt = this.unreachableQuarantine.get(peerID); + if (quarantinedAt !== undefined) { + if (Date.now() - quarantinedAt < UNREACHABLE_QUARANTINE_MS) { + trace(`[NET] addBootstrapPeers skip quarantined: ${peerID.slice(0, 16)}`); + continue; + } + this.unreachableQuarantine.delete(peerID); + // This dial is the ONE probe an expired quarantine buys. If it fails the + // window has to close again — otherwise every later gossip mention spends + // another dial and refreshes the status row, which is exactly the churn + // the quarantine exists to stop. + probeAfterQuarantine = true; + } + } + // Single-flight, keyed by the endpoint rather than the peer: two addresses of + // one peer are two different questions and both deserve their own dial, while + // two runs asking about the SAME address duplicate a 10 s timeout for one + // answer. Claimed after every skip above so a refused candidate never blocks + // the run that would actually dial it. + if (inFlight.has(canonicalAddress)) { + trace(`[NET] addBootstrapPeers skip in-flight: ${peer}`); + continue; + } + inFlight.add(canonicalAddress); + // A CONFIGURED identity is user data and enters the set on the strength of the + // saved config alone. A DISCOVERED one waits for the dial: it arrived in a + // gossip message and nothing has yet shown that the identity exists, let + // alone that it is the one behind this address. Admitting it here put every + // peer ID any topic subscriber cared to name into an unbounded global set — + // one that nothing prunes, and that other code reads as "this peer is + // handled". Deduplication of repeated mentions is not this set's job and + // never was: {@link inFlightBootstrapDials} and the backoff above do that. + if (peerID && origin === 'configured') this.bootstrapPeerIDs.add(peerID); console.debug('Adding bootstrap peer:', peer); - this.bootstrapTracker.markPending(networkID, peer, peerID, origin); + this.bootstrapTracker.markPending(networkID, peer, peerID, effectiveOrigin); try { - // Skip re-dialing when libp2p already has an active connection to this peer - // (typical when the same bootstrap entry appears in multiple lishnets). - // We still record the outcome so per-network status reflects "connected" - // rather than leaving the entry stuck at "pending". - const reuseExisting = alreadyKnown && peerID && this.node.getConnections(peerIDFromString(peerID)).length > 0; - if (!reuseExisting) await this.node.dial(ma); - if (peerID) { - await this.node.peerStore.merge(peerIDFromString(peerID), { - multiaddrs: [ma], - tags: { [KEEP_ALIVE]: { value: 1 } }, - }); + // Always hand the address to libp2p and let IT decide whether a dial is + // needed. Skipping the call whenever any connection to the peer existed + // was too coarse: libp2p reuses only a DIRECT, unlimited connection, and + // deliberately dials when it holds a relayed one and the new address + // would upgrade it to direct. Pre-empting that cost us the upgrade, and + // left a bad configured address permanently untested — its identity + // mismatch undiscovered — whenever the peer happened to be reachable + // some other way. + // + // The address may still enter the address book only when it is + // Noise-verified, otherwise a topic subscriber could poison a connected + // peer's addresses with entries that later feed re-dials and eviction. + // Verification is now read off the RESULT: the connection libp2p handed + // back is proof for `ma` only if that is the address it is actually on. + // A configured address is the user's own claim and its status row is how + // they debug it, so it gets a real probe: `force` makes libp2p contact + // THIS address instead of handing back a connection it already holds to + // the same peer, which is what let a broken configured entry sit there + // showing "connected" — and kept its identity mismatch undiscovered — + // merely because the peer was reachable some other way. + // + // Discovered addresses never force: they arrive from gossip, and a peer + // that names many of them could otherwise make us open a connection per + // address. For those, libp2p's own reuse is the desired behaviour. + const pidObj = peerID ? peerIDFromString(peerID) : null; + const conn = await this.node.dial(ma, effectiveOrigin === 'configured' ? { force: true } : {}); + const verifiedThisAddr = isSameDialEndpoint(String(conn?.remoteAddr ?? ''), ma.toString()); + // A dial already in flight cannot be called back: hangUp only closes + // connections that ALREADY exist, so a leave-network landing mid-dial finds + // nothing to close and this connection surfaces a moment after the cleanup + // finished. Abandoning the loop would leave it open, so close it here — the + // suppression set is what says the user deliberately left this peer. + // Two ways this connection can already be unwanted: the peer was hung up + // by leave-network and sits in the suppression set, or we left the network + // this dial belonged to before ever seeing the peer as one of its members, + // in which case it never entered that set at all. + // + // The EPOCH is checked first and separately from the full `superseded()`: + // a different node instance means none of this run's suppression or + // subscription state describes the node that would be torn at, so the + // destructive branch must not run at all. A bumped GENERATION is the + // opposite case — the network was left or its list replaced on the SAME + // node, which is precisely when this connection needs closing, so it may + // not short-circuit ahead of it. + if (epoch !== this.runEpoch) return; + if (peerID && networkID && (this.isRedialSuppressed(peerID) || !this.isTopicSubscribed(networkID)) && !this.isPeerNeededByJoinedNetwork(peerID)) { + trace(`[NET] bootstrap dial landed after leave, disconnecting: ${peerID.slice(0, 16)}`); + await this.disconnectPeer(peerID, networkID, epoch); + return; + } + if (superseded()) return; + // The peer answered, so the identity behind this address is real and + // wanted — the point at which a discovered ID has earned its place in + // the set (see the claim above for why it may not have it yet). + if (peerID) this.bootstrapPeerIDs.add(peerID); + // A CONFIGURED bootstrap joins the gossipsub direct set the moment it + // answers. Production starts the node with an empty list — the config-time + // `directPeers` seed never applies — so waiting for the periodic promotion + // left the peer the whole mesh depends on without a fast reconnect, and + // PRUNE-able, for the first ~150 s of every run. + if (peerID && effectiveOrigin === 'configured') this.addGossipsubDirectPeer(peerID); + if (pidObj) { + await this.node.peerStore.merge(pidObj, verifiedThisAddr ? { multiaddrs: [ma], tags: { [KEEP_ALIVE]: { value: 1 } } } : { tags: { [KEEP_ALIVE]: { value: 1 } } }); } - this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, null, origin); + // Re-check after the merge await too: stop() may have cleared the + // tracker while it was pending, and recordOutcome would otherwise + // resurrect a network row for the old (or next) node instance. + if (superseded()) return; + // `force: true` defeats connection REUSE, but not a dial to the same peer + // ID already in libp2p's queue: this call joins that job and can be handed + // the connection its other address won. For a CONFIGURED entry the row + // means "this address works", so an unverified dial must leave it pending + // rather than turn it green — a wrong address that the peer happens to + // survive through another route is exactly what the row exists to expose. + // Discovered rows carry the weaker "the peer answered" meaning and are + // recorded either way. + if (effectiveOrigin === 'configured' && !verifiedThisAddr) { + trace(`[NET] bootstrap addr unverified (connection came back on another address), left pending: ${peer}`); + continue; + } + // A gossip-learned address has now answered on the endpoint it claimed, so + // it has earned its place in the autodial list. Unverified ones never get + // there, which is what keeps a flood of invented addresses off it. + if (effectiveOrigin === 'discovered' && verifiedThisAddr) this.rememberBootstrapAddress(ma); + // The identity Noise actually proved on this connection, not the one the + // address claimed. It is the only evidence the row-cap ranking accepts, and + // passing null here left an active, verified member ranked as an ordinary + // connected row — evictable by age alongside the invented addresses the + // ranking exists to drop. + this.bootstrapTracker.recordOutcome(networkID, peer, peerID, 'connected', null, conn?.remotePeer?.toString() ?? null, effectiveOrigin); console.log('✓ Connected to new bootstrap peer'); } catch (err: any) { + if (superseded()) return; const message = err?.message ?? String(err); const kind = classifyBootstrapError(message); + // The probe the expired quarantine allowed has failed, so close the window + // again rather than letting the next announce buy another dial. + if (probeAfterQuarantine && peerID) this.unreachableQuarantine.set(peerID, Date.now()); + // Pay the failure into the shared per-peer backoff the check above reads. + // Without this the check could never bite for a gossip-learned peer: nothing + // else writes that record for a peer absent from the peerStore, so re-dial + // maintenance never sees it either and every announce bought another dial. + // Same accounting as the recovery loop — pacing only, no eviction credit, + // since a failed announce dial says nothing about who is the broken side. + if (peerID && effectiveOrigin === 'discovered') this.noteRecoveryDialFailure(peerID); const actualPeerID = kind === 'identity-mismatch' ? extractActualPeerID(message) : null; - this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, origin); + this.bootstrapTracker.recordOutcome(networkID, peer, peerID, kind, message, actualPeerID, effectiveOrigin); // [NET-MISMATCH] richer log for identity-mismatch — single line containing // origin (configured / discovered from peer-announce), multiaddr, // expected peerID and the actual peerID Noise reported. Makes it // trivial to grep `[NET-MISMATCH]` and diff what the catalog has // vs reality, even before the UI shows the same data. if (kind === 'identity-mismatch') { - console.log(`[NET-MISMATCH] origin=${origin} net=${networkID?.slice(0, 8) ?? 'none'} addr=${peer} expected=${peerID ?? 'none'} actual=${actualPeerID ?? 'unparsed'}`); + console.log(`[NET-MISMATCH] origin=${effectiveOrigin} net=${networkID?.slice(0, 8) ?? 'none'} addr=${peer} expected=${peerID ?? 'none'} actual=${actualPeerID ?? 'unparsed'}`); } else { console.log(`⚠️ Could not connect to bootstrap peer (${kind}): ${peer} — ${message}`); } - // Crypto-verified identity mismatch ⇒ peerID stored in our peerStore - // is provably wrong for this address. Purge it so libp2p autodial - // stops retrying the dead identity. Safe because Noise handshake - // is unforgeable — a mismatch is definitive, never a transient - // network issue. Only triggers when we have an expected peerID - // to purge. + // Crypto-verified identity mismatch ⇒ THIS ADDRESS provably no longer + // belongs to the expected peer (Noise is unforgeable). It says nothing + // about the peer's other addresses: a peer healthy over a relay can + // still have one stale direct address that some other node now owns. + // So: peer alive through other connections → drop only the offending + // address; peer with no connections → full purge as before. if (kind === 'identity-mismatch' && peerID) { - await this.purgeStalePeer(peerID, `${origin} dial identity mismatch`); + const pid = peerIDFromString(peerID); + // Compare in a form that survives both multiaddr normalization (expanded → + // compressed IPv6) and DNS case / trailing-dot differences — otherwise a + // filter that fails to match silently keeps the poisoned address while + // logging that it was dropped. + const canonical = normalizeMultiaddrForCompare(ma.toString()); + const canonicalBare = canonical.replace(/\/p2p\/[^/]+$/, ''); + const matches = (str: string): boolean => { + const n = normalizeMultiaddrForCompare(str); + return n === canonical || n === canonicalBare; + }; + // Noise proves exactly one thing: THIS address no longer leads to the peer + // we expected. It says nothing about the peer's other addresses, so the bad + // one goes first and unconditionally — whether or not the peer happens to be + // connected right this moment. Purging on "not currently connected" threw + // away addresses that were never disproved. + // Restrict the autodial-list filter to entries of THIS peer so a + // case-insensitive compare can never drop a different peer's addr. + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(m => extractDestinationPeerID(m) !== peerID || !matches(m.toString())); + let remainingAddresses = 0; + try { + const rec = await this.node.peerStore.get(pid); + const keep = rec.addresses.filter((a: any) => !matches(a.multiaddr.toString())); + if (keep.length < rec.addresses.length) await this.node.peerStore.patch(pid, { multiaddrs: keep.map((a: any) => a.multiaddr) }); + remainingAddresses = keep.length; + } catch { + /* peer not in store — nothing to trim, and nothing left either */ + } + if (superseded()) return; + // Only once the peer has neither a live connection nor a single address we + // have not disproved is there anything left to purge. + if (this.node.getConnections(pid).length === 0 && remainingAddresses === 0) { + await this.purgeStalePeer(peerID, `${effectiveOrigin} dial identity mismatch, no usable address left`, epoch); + } else { + console.log(`[NET] dropped stale addr of peer ${peerID.slice(0, 16)}: ${ma.toString()}`); + } // For DISCOVERED entries (peer-announce gossip), also drop the // status entry — there's no saved config row to "fix" and leaving // it visible just adds UI noise. For CONFIGURED entries, keep // it so the user can decide to update or remove the saved row. - if (origin === 'discovered' && networkID) { + if (effectiveOrigin === 'discovered' && networkID) { this.bootstrapTracker.deletePeer(networkID, peer); } } + } finally { + // Every exit from the dial block releases the claim, `return` included — + // a leave landing mid-dial would otherwise lock the address out for the + // lifetime of the node. + inFlight.delete(canonicalAddress); } } catch (error: any) { this.bootstrapTracker.recordOutcome(networkID, peer, null, 'error', error?.message ?? String(error), null, origin); @@ -1137,6 +2050,19 @@ export class Network { this.bootstrapTracker.setOnChange(cb); } + /** + * True when this node holds a live connection to somebody OTHER than the given + * peer. + * + * This is the difference between "that peer is gone" and "we are the ones who + * are offline", and eviction is only ever entitled to the first reading. The + * peer being judged is excluded because a connection to it would make the + * question moot — that case is handled separately, right before the purge. + */ + private hasConnectionOtherThan(peer: PeerID): boolean { + return !!this.node && this.node.getConnections().some(connection => !connection.remotePeer.equals(peer)); + } + /** * Remove a peerID from libp2p's peerStore + drop it from our bootstrap dedup set. * @@ -1147,14 +2073,30 @@ export class Network { * * Best-effort: a peerStore.delete failure is logged at debug but does not throw — * the same peer will be re-purged next cycle if libp2p keeps trying it. + * + * `epoch` binds the call to the node instance it was started for. This is the most + * destructive path there is — it closes connections and deletes peerStore entries — + * and it awaits in the middle, so a stop()/start() landing between those awaits + * would otherwise let it finish against the NEXT node and evict a peer that + * instance never had a problem with. The node reference is captured once for the + * same reason: re-reading `this.node` after an await can hand back a different node. */ - async purgeStalePeer(peerID: string, reason: string): Promise { - if (!this.node) return; + async purgeStalePeer(peerID: string, reason: string, epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; this.bootstrapPeerIDs.delete(peerID); + // Drop the peer's addrs from the autodial list too — this array is otherwise + // push-only, so the zero-connection recovery loop would keep dialing addrs + // of an identity we just proved dead, and the array would grow until stop(). + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => extractDestinationPeerID(ma) !== peerID); + // Remove from the gossipsub never-PRUNE direct set, or gossipsub keeps + // attempting a direct stream to the dead peer every directConnectTicks. + this.removeGossipsubDirectPeer(peerID); + this.redialBackoff.delete(peerID); try { const pid = peerIDFromString(peerID); // Drop existing connections so libp2p considers the entry fully gone. - const conns = this.node.getConnections(pid); + const conns = node.getConnections(pid); for (const c of conns) { try { await c.close(); @@ -1162,13 +2104,71 @@ export class Network { /* connection may already be closing */ } } - await this.node.peerStore.delete(pid); + // Closing connections yields; bail before the irreversible delete if this + // run no longer owns the node. + if (epoch !== this.runEpoch) return; + await node.peerStore.delete(pid); console.log(`[NET] purged stale peerStore entry ${peerID.slice(0, 16)}… (reason: ${reason})`); + // TOCTOU healing: an inbound connection can land between the caller's + // liveness check and the delete above. The peer:connect handler resets + // failure counters but cannot restore the bootstrap/keep-alive state this + // purge just removed — so if the peer is connected NOW, rebuild its dial + // state from the live connections; otherwise reconnect would silently die + // with the first drop. + // + // Not for a peer leave-network hung up: it is meant to be forgotten, and a + // connection racing the purge is not a reason to rebuild what the leave + // deliberately tore down. + if (epoch !== this.runEpoch) return; + const after = node.getConnections(pid); + if (after.length > 0 && !this.isRedialSuppressed(peerID)) { + await this.restorePurgedPeerState(node, pid, after, epoch); + } } catch (err: any) { trace(`[NET] purgeStalePeer ${peerID.slice(0, 16)} failed: ${err?.message ?? err}`); } } + /** + * Put back everything {@link purgeStalePeer} took away, for a peer that turns out to + * be connected after all. + * + * The purge removes four things — the bootstrap dedup entry, the peer's addresses + * from the autodial list, its gossipsub direct entry and its keep-alive tag — and + * restoring only some of them left a state nothing else repairs: periodic promotion + * skips any peer already in `bootstrapPeerIDs`, so the missing address and direct + * entry would stay missing for as long as the peer stayed connected, and the next + * drop would find no way back. + * + * `bootstrapPeerIDs` is therefore filled in LAST. It is the flag the other paths read + * as "this peer is handled"; setting it first is what let promotion observe a + * half-restored peer and walk away from it. + */ + private async restorePurgedPeerState(node: Libp2p, pid: PeerID, connections: Array<{ remoteAddr: any }>, epoch: number): Promise { + const peerID = pid.toString(); + this.unreachableQuarantine.delete(peerID); + await node.peerStore.merge(pid, { + multiaddrs: connections.map(c => c.remoteAddr), + tags: { [KEEP_ALIVE]: { value: 1 } }, + }); + if (epoch !== this.runEpoch) return; + for (const c of connections) { + // The autodial list is walked by peer ID, so an address that does not already + // end in this peer's identity gets the suffix — the same shape promotion builds. + const remote = c.remoteAddr; + if (!remote) continue; + try { + this.rememberBootstrapAddress(extractDestinationPeerID(remote) === peerID ? remote : Multiaddr(`${remote.toString()}/p2p/${peerID}`)); + } catch { + // Unparseable remote address — nothing to put back on the list for it. + } + } + const gossipsub: any = this.pubsub; + if (gossipsub?.direct && typeof gossipsub.direct.add === 'function') gossipsub.direct.add(peerID); + this.bootstrapPeerIDs.add(peerID); + console.log(`[NET] purge raced an inbound connection — restored ${peerID.slice(0, 16)}…`); + } + /** * True if the peer is one we must never voluntarily disconnect because it * provides infrastructure rather than being a plain content peer: an @@ -1189,10 +2189,87 @@ export class Network { * lishnet layer when a bootstrap entry is removed from config or belongs only * to a lishnet being left, so `isBootstrapOrRelayPeer` stops treating a peer * that is no longer configured (nor shared with another joined network) as - * infrastructure that leave-network must keep connected. + * infrastructure that leave-network must keep connected — and so the + * unreachable-eviction exemption ends with it. + * + * Both callers already establish that the peer is configured in NO joined + * network before calling, so this needs no refcount of its own. */ + /** + * Put an address on the autodial list that zero-connection recovery walks, unless + * it is already there. + * + * Membership is decided by the ADDRESS, not by the peer ID behind it: a bootstrap + * whose host or port the user edited keeps its identity, and an identity-keyed + * check would treat the new address as already known and never add it — leaving + * recovery dialing the address that was replaced. + */ + private rememberBootstrapAddress(ma: any): void { + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (this.bootstrapMultiaddrs.some(m => normalizeMultiaddrForCompare(m.toString()) === canonical)) return; + this.bootstrapMultiaddrs.push(ma); + if (this.bootstrapMultiaddrs.length <= MAX_BOOTSTRAP_ADDRESSES) return; + // Array order is insertion order, so the first discovered entry is the oldest one. + // A list of nothing but configured entries is left to grow: it is bounded by what + // the user saved, and dropping any of it would silently unconfigure a bootstrap. + const oldestDiscovered = this.bootstrapMultiaddrs.findIndex(m => !this.configuredBootstrapAddresses.has(normalizeMultiaddrForCompare(m.toString()))); + if (oldestDiscovered === -1) return; + trace(`[NET] autodial list full — dropping ${this.bootstrapMultiaddrs[oldestDiscovered]?.toString()}`); + this.bootstrapMultiaddrs.splice(oldestDiscovered, 1); + } + + /** + * Take specific addresses off the autodial list. Used when a network's configured + * list changes: an entry that is gone must stop being dialed, and that includes + * the case where the peer ID stays and only its address moved, which + * {@link pruneConfiguredBootstrapPeer} cannot see because the identity is still + * configured. + */ + pruneBootstrapAddresses(addresses: string[]): void { + if (addresses.length === 0) return; + const drop = new Set(addresses.map(a => normalizeMultiaddrForCompare(a))); + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => !drop.has(normalizeMultiaddrForCompare(ma.toString()))); + for (const address of drop) { + this.configuredBootstrapAddresses.delete(address); + // The pacing record goes with the address it paces. Left behind, it accumulates + // across every configuration change until stop(), and — worse — a re-added + // address inherits the old failCount and its multi-minute nextAttempt, so a user + // who deletes an entry and puts it back may see nothing dialed for minutes. + this.addressProbeBackoff.delete(address); + } + } + pruneConfiguredBootstrapPeer(peerID: string): void { this.configuredBootstrapPeerIDs.delete(peerID); + // Symmetric with what the configured lifecycle hands out on the way in — a direct-set + // entry and a KEEP_ALIVE tag, both granted the moment the entry answers a dial. + // Without the matching removal, a bootstrap edited out of the list kept its fast + // reconnect cadence and its eviction exemption for the rest of the run: gossipsub + // went on opening a direct stream to it every directConnectTicks and libp2p went on + // re-dialing it, so the peer the user deleted never actually went away. + this.removeGossipsubDirectPeer(peerID); + this.clearBootstrapKeepAlive(peerID); + // Forget its addresses too. They were pushed into the autodial list when the + // entry was first configured, and that list is what zero-connection recovery + // walks — leaving them there means a bootstrap the user has just deleted keeps + // being dialed whenever the node runs out of connections, which is exactly the + // churn this work removes. The dedup set has to let go as well, or a later + // re-add would be treated as already known and the address could never come back. + this.bootstrapPeerIDs.delete(peerID); + // Only the addresses that came from the config. The same peer may also have a + // gossip-learned address that earned its place by answering a dial — that one + // belongs to the discovered lifecycle (TTL, backoff) and is not the user's to lose + // just because they deleted a different address of the same peer. + this.bootstrapMultiaddrs = this.bootstrapMultiaddrs.filter(ma => { + if (extractDestinationPeerID(ma) !== peerID) return true; + const canonical = normalizeMultiaddrForCompare(ma.toString()); + if (!this.configuredBootstrapAddresses.has(canonical)) return true; + this.configuredBootstrapAddresses.delete(canonical); + // Same reason as in pruneBootstrapAddresses: the pacing record belongs to the + // address, so a re-add must not inherit the deleted entry's backoff. + this.addressProbeBackoff.delete(canonical); + return false; + }); } isBootstrapOrRelayPeer(peerID: string): boolean { @@ -1243,6 +2320,28 @@ export class Network { * LISHs just because a transport connection exists (e.g. the peer's * keep-alive re-dialed us right after we left its network). */ + /** + * Whether a lishnet we are STILL in has any claim on this peer. + * + * Leaving one network says nothing about the others — the same peer can be a member + * of a second lishnet or its configured bootstrap. Tearing it down is destructive + * (disconnectPeer suppresses re-dials AND drops the peerStore entry), so that is + * reserved for a peer no joined network has a use for. It is the same question + * leaveNetwork asks before it hangs anyone up. + */ + private isPeerNeededByJoinedNetwork(peerID: string): boolean { + if (this.isBootstrapOrRelayPeer(peerID)) return true; + if (this.sharesJoinedTopicWith(peerID)) return true; + if (!this.pubsub) return false; + for (const topic of this.pubsub.getTopics()) { + if (!topic.startsWith(LISH_TOPIC_PREFIX)) continue; + // Recently-seen counts as well: a member that is momentarily disconnected is + // still a member, and leaveNetwork widens its own snapshot the same way. + if (this.peerAnnounce.getRecentMembers(topic).includes(peerID)) return true; + } + return false; + } + sharesJoinedTopicWith(peerID: string): boolean { if (!this.pubsub) return false; for (const topic of this.pubsub.getTopics()) { @@ -1328,9 +2427,15 @@ export class Network { * * `networkID` is the lishnet the peer is being left with — the peer is suppressed * under it so rejoining that lishnet lifts exactly its peers. + * + * `epoch` binds the whole sequence to the node instance it started on, for the same + * reason {@link purgeStalePeer} takes one: this awaits twice, and re-reading + * `this.node` after a stop()/start() used to hand back the NEW node — so the hangUp + * and the purge landed on an instance that had never heard of this leave. */ - async disconnectPeer(peerID: string, networkID: string): Promise { - if (!this.node) return; + async disconnectPeer(peerID: string, networkID: string, epoch: number = this.runEpoch): Promise { + const node = this.node; + if (!node || epoch !== this.runEpoch) return; let pid: PeerID; try { pid = peerIDFromString(peerID); @@ -1338,6 +2443,13 @@ export class Network { trace(`[NET] disconnectPeer: invalid peerID ${peerID.slice(0, 16)}: ${err?.message ?? err}`); return; } + // Suppression is claimed BEFORE the first await, not after the hangUp. The two + // awaits below yield, and a `peer:discovery` event landing in that window used to + // read "not suppressed", start a dial, and have it complete after the hangUp had + // already searched for connections and found none — leaving the peer connected + // with the leave apparently finished. Recording the intent up front makes the + // window harmless: the dial that lands late sees the suppression and closes itself. + this.addRedialSuppression(networkID, peerID); // Remove the keep-alive tags FIRST so the imminent hangUp does not race // the ReconnectQueue back into a re-dial. Both tags matter: the custom // 'keep-alive-fleet' tag (peer-announce intake) and the native KEEP_ALIVE @@ -1347,25 +2459,22 @@ export class Network { // undefined as the tag value removes it (per @libp2p/interface PeerStore // merge semantics). try { - await this.node.peerStore.merge(pid, { + await node.peerStore.merge(pid, { tags: { 'keep-alive-fleet': undefined, [KEEP_ALIVE]: undefined }, }); } catch (err: any) { trace(`[NET] disconnectPeer: tag removal failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } + if (epoch !== this.runEpoch) return; try { - await this.node.hangUp(pid); + await node.hangUp(pid); trace(`[NET] disconnectPeer: hung up ${peerID.slice(0, 16)}`); } catch (err: any) { trace(`[NET] disconnectPeer: hangUp failed for ${peerID.slice(0, 16)}: ${err?.message ?? err}`); } - // Keep redial maintenance from re-dialing this just-left peer on the next - // status tick. Keyed by the left lishnet so rejoin lifts exactly its peers; - // cleared automatically once it reconnects legitimately. - this.addRedialSuppression(networkID, peerID); // Forget the persisted peerStore entry so the disconnect survives a restart — // suppression is in-memory only, but the peerStore is on disk. - await this.purgeStalePeer(peerID, 'left-network exclusive peer'); + await this.purgeStalePeer(peerID, 'left-network exclusive peer', epoch); } /** Snapshot of all per-network bootstrap statuses. */ @@ -1380,25 +2489,50 @@ export class Network { /** Drop bootstrap status entries no longer in the configured peer list (after an update). */ pruneBootstrapStatus(networkID: string, keepMultiaddrs: string[]): void { + this.bumpBootstrapGeneration(networkID); this.bootstrapTracker.pruneEntries(networkID, keepMultiaddrs); } /** Reset the bootstrap status for a single network (used when re-joining). */ resetBootstrapStatus(networkID: string): void { + this.bumpBootstrapGeneration(networkID); this.bootstrapTracker.resetNetwork(networkID); } + /** + * Current bootstrap-config version of a network. Entries with no network (the + * startup catalog dial) share version 0 and are only bound by the run epoch. + */ + private bootstrapGenerationOf(networkID: string | null): number { + return networkID === null ? 0 : (this.bootstrapGeneration.get(networkID) ?? 0); + } + + /** + * Declare a network's configured bootstrap list superseded, abandoning any + * `addBootstrapPeers` job still walking the previous one. Called whenever that + * list is replaced, reset or left — see the generation comment in + * {@link addBootstrapPeers} for what an unabandoned job would resurrect. + */ + bumpBootstrapGeneration(networkID: string): void { + this.bootstrapGeneration.set(networkID, this.bootstrapGenerationOf(networkID) + 1); + } + // ========================================================================= // Topic (lishnet) management // ========================================================================= /** * Subscribe to a lishnet topic. The node will receive pubsub messages for this network. + * + * Returns whether the subscription actually happened. Reporting nothing let the caller + * record a network as joined after a no-op on a stopped node: `joinedNetworks` claiming + * membership of a topic nobody is subscribed to, which a later rejoin then treated as + * "already joined" and skipped. */ - subscribeTopic(networkID: string): void { + subscribeTopic(networkID: string): boolean { if (!this.pubsub) { console.error('Network not started - cannot subscribe to topic'); - return; + return false; } const topic = lishTopic(networkID); this.pubsub.subscribe(topic); @@ -1467,9 +2601,8 @@ export class Network { this.topicHandlers.get(topic)!.add(handler); console.log(`✓ Subscribed to lishnet topic: ${topic}`); // GossipSub mesh needs time to rebuild after subscribe — schedule delayed peer count checks - setTimeout(() => this.schedulePeerCountCheck(), 2000); - setTimeout(() => this.schedulePeerCountCheck(), 5000); - setTimeout(() => this.schedulePeerCountCheck(), 15000); + for (const delay of [2000, 5000, 15000]) this.armDelayedPeerCountCheck(delay); + return true; } /** @@ -1480,6 +2613,18 @@ export class Network { if (handlers) handlers.delete(handler); } + /** + * Whether this node is still subscribed to a lishnet's topic, i.e. has not left it. + * + * Without pubsub there is no answer to give, and the caller uses this to decide + * whether to tear a connection down — so the unknown case reports "still joined", + * which is the harmless one. + */ + isTopicSubscribed(networkID: string): boolean { + if (!this.pubsub) return true; + return this.pubsub.getTopics().includes(lishTopic(networkID)); + } + unsubscribeTopic(networkID: string): void { if (!this.pubsub) return; const topic = lishTopic(networkID); @@ -1602,9 +2747,23 @@ export class Network { console.error('Network not started'); return; } + await Network.publishOn(this.pubsub, topic, data); + } + + /** + * Publish over a SPECIFIC pubsub instance rather than whatever `this.pubsub` is now. + * + * A long-running emitter captures the pubsub of the run it started in and then awaits + * — repeatedly, once per topic. Routing those later publishes through + * {@link broadcast} re-reads the field, so a stop/start landing in one of the awaits + * put the OLD run's payload (its identity, its addresses, its peerStore snapshot) onto + * the NEW node's topics. Binding the publish to the captured transport makes that + * impossible instead of merely unlikely. + */ + static async publishOn(pubsub: any, topic: string, data: Record): Promise { trace(`[NET] broadcast ${topic}: ${data['type']}`); const encoded = new TextEncoder().encode(JSON.stringify(data)); - const result = await this.pubsub.publish(topic, encoded); + const result = await pubsub.publish(topic, encoded); const recips = (result as any)?.recipients?.map((p: any) => p.toString().slice(0, 12)) ?? []; trace(`[NET] broadcast ${topic.slice(0, 28)}: ${data['type']} → recipients=[${recips.join(',')}] count=${recips.length}`); } @@ -1660,13 +2819,29 @@ export class Network { return { peerID: this.node.peerId.toString(), privateKeyBytes: bytes }; } + /** + * Run a destructive identity/datastore operation with the run provably over. + * + * `if (this.node) throw` was not that check. For the whole first half of + * {@link startLocked} the datastore file is already open and the identity already + * read while `this.node` is still null, so the guard passed and the wipe landed + * underneath an in-progress start — a deleted peerstore mid-open, or an identity + * overwritten after it was read but before the node existed. Holding the same + * lifecycle mutex and demanding `stopped` inside it covers every await of a start. + */ + private async runWhenStopped(what: string, op: () => Promise): Promise { + return await this.lifecycleMutex.runExclusive(async () => { + if (this.lifecycle !== 'stopped') throw new CodedError(ErrorCodes.INTERNAL_ERROR, `Network must be stopped before ${what}`); + return await op(); + }); + } + /** * Write a new identity private key into the datastore. The network must be stopped. * Validates the protobuf bytes by attempting to decode them. */ async writeIdentityKey(privateKeyBytes: Uint8Array): Promise { - if (this.node) throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network must be stopped before writing identity key'); - await writeIdentityKeyToDatastore(this.dataDir, privateKeyBytes); + await this.runWhenStopped('writing identity key', () => writeIdentityKeyToDatastore(this.dataDir, privateKeyBytes)); } /** @@ -1674,8 +2849,7 @@ export class Network { * Next start will generate a fresh key. */ async clearIdentityKey(): Promise { - if (this.node) throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network must be stopped before clearing identity key'); - await clearIdentityKeyFromDatastore(this.dataDir); + await this.runWhenStopped('clearing identity key', () => clearIdentityKeyFromDatastore(this.dataDir)); } /** @@ -1684,8 +2858,7 @@ export class Network { * fresh identity and an empty peerstore. Used by the factory reset. */ async clearDatastore(): Promise { - if (this.node) throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network must be stopped before clearing datastore'); - await clearDatastoreDir(this.dataDir); + await this.runWhenStopped('clearing datastore', () => clearDatastoreDir(this.dataDir)); } /** @@ -1694,8 +2867,7 @@ export class Network { * but discovers peers fresh. Used by the factory reset "peers" category. */ async clearPeerstore(): Promise { - if (this.node) throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network must be stopped before clearing peerstore'); - await clearPeerstoreOnly(this.dataDir); + await this.runWhenStopped('clearing peerstore', () => clearPeerstoreOnly(this.dataDir)); } /** @@ -1730,10 +2902,53 @@ export class Network { } async stop(): Promise { + // Same mutex as start(): a stop that overlapped a start used to tear down a node + // the start had just handed back to its caller as successfully started. + await this.lifecycleMutex.runExclusive(async () => { + // An interrupted libp2p stop cannot be resumed, so a retry would do nothing and + // report success — the exact no-op that let a half-stopped node be treated as + // down. Refusing is the honest answer; the process has to be restarted. + if (this.nodeStopUnrecoverable) throw new CodedError(ErrorCodes.INTERNAL_ERROR, 'Network is in a terminal failed state: its libp2p node could not be stopped and cannot be stopped again — restart the process'); + this.lifecycle = 'stopping'; + try { + await this.teardown(); + this.lifecycle = 'stopped'; + } catch (err) { + // teardown kept whatever it could not prove released. Setting `stopped` here + // regardless is what let the caller go on to start a second node over the same + // identity, port and datastore, and let a factory reset wipe a datastore still + // in use. A retry of stop() repeats only the phases that are still outstanding. + this.lifecycle = 'failed'; + throw err; + } + }); + } + + /** + * Release everything a run owns. Shared by {@link stop} and by the failure path of + * {@link start}, because a half-built start holds the same resources a finished one + * does — and leaving them behind is what made a failed start unrecoverable. + * + * The per-run bookkeeping below is cleared unconditionally, but each owned resource is + * released only once it is provably gone: the node, the pubsub handle and the identity + * after libp2p has reached `stopped`, the datastore after its own close returned. A stop + * that could not prove the node down has not shown it to be down, and everything that + * follows — closing its datastore, dropping the reference, permitting a new start or a + * wipe — is only safe once it has. That failure propagates and leaves the instance in + * `failed`; because libp2p cannot resume an interrupted stop, that particular `failed` + * is permanent and {@link stop} refuses to pretend otherwise. + */ + private async teardown(): Promise { + this.runEpoch++; // invalidate any in-flight status tick before touching state if (this.statusInterval) { clearInterval(this.statusInterval); this.statusInterval = null; } + // The epoch bump makes an in-flight tick bail out, but its `finally` runs + // asynchronously — a fast restart would otherwise find the flag still set and + // skip its own first tick. The tick only clears the flag for its own epoch, so + // clearing it here cannot be undone by the outgoing run. + this.statusTickInFlight = false; this.peerAnnounce.stop(); if (this.wantResponseCleanupInterval) { clearInterval(this.wantResponseCleanupInterval); @@ -1762,23 +2977,70 @@ export class Network { this.bootstrapPeerIDs.clear(); this.bootstrapTracker.clear(); this.bootstrapMultiaddrs = []; + this.bootstrapGeneration.clear(); + // Claims belong to the node being torn down; a dial still settling on the old node + // must not lock the address out for the next one — nor, by releasing into a shared + // Set, steal the claim the next one has taken. A new object does both. + this.inFlightBootstrapDials = new Set(); + this.inFlightDiscoveryDials = new Set(); this._lastPeerCounts.clear(); this._lastScores.clear(); this.redialBackoff.clear(); + this.unreachableQuarantine.clear(); + this.addressProbeBackoff.clear(); + this.noReachableSince.clear(); + this.configuredBootstrapPeerIDs.clear(); + this.configuredBootstrapAddresses.clear(); + // Per-run like the rest of this state: a fresh node must not inherit a count that + // makes its very first tick the slow-cadence one. + this.statusTickCount = 0; + for (const timer of this.delayedPeerCountTimers) clearTimeout(timer); + this.delayedPeerCountTimers.clear(); this.redialSuppressedByNet.clear(); this.pxIngressLogKeys.clear(); - if (this.node) { - await this.node.stop(); - console.log('Network stopped'); - } - if (this.datastore) { - await this.datastore.close(); - console.log('Datastore closed'); + try { + if (this.node) { + await this.node.stop(); + // A resolved stop() is NOT proof the node is down. libp2p sets `status` to + // 'stopping' before its own stop phases and to 'stopped' only after all of them + // returned; a phase that throws leaves the status at 'stopping' permanently, and + // every later stop() call sees a status that is not 'started' and returns at once + // without doing any more work. Taking that silent no-op for success is what let a + // node still holding its listener, its connections and its port be reported down. + if (this.node.status !== 'stopped') throw new CodedError(ErrorCodes.INTERNAL_ERROR, `libp2p did not reach 'stopped' (status: ${this.node.status}) and cannot resume an interrupted stop`); + console.log('Network stopped'); + } + } catch (err: any) { + trace(`[NET] node.stop() failed: ${err?.message ?? err}`); + // A node that refused to stop may still hold its listener, its connection + // manager and its port. Closing the datastore it is working over, and dropping + // the last reference to it, made the damage permanent AND invisible: nobody + // could see the shutdown had not happened, and the caller was free to start a + // second node over the same identity, port and datastore. Keep both. + this.nodeStopUnrecoverable = true; + throw err; } + // Released only now, with the node provably down: everything below and everything a + // later start or wipe may do is safe only once it is. this.node = null; this.pubsub = null; - this.datastore = null; this.currentPrivateKey = null; + try { + if (this.datastore) { + await this.datastore.close(); + console.log('Datastore closed'); + } + } catch (err: any) { + trace(`[NET] datastore.close() failed: ${err?.message ?? err}`); + // SqliteDatastore.close() closes the database handle directly and can throw. + // Swallowing that and reporting a clean stop lost the reference to a database + // still open: the close could never be retried, a new start would open a second + // handle on the same file, and clearDatastore / clearPeerstore / an identity + // change were all permitted over it. The node above is provably down and stays + // released, so a retried stop() repeats only this close. + throw err; + } + this.datastore = null; } async cliFindPeer(peerID: string): Promise { @@ -1801,6 +3063,39 @@ export class Network { } } +// Re-exported so callers and tests that already reach for this here keep working; the +// implementation lives in multiaddr-utils so network-config can share it without +// importing this module. +export { extractDestinationPeerID }; + +/** + * Normalize a multiaddr STRING for equality comparison. + * + * Delegates to {@link canonicalMultiaddr}, which parses the address first — that is + * what folds an expanded IPv6 literal into its compressed form. Doing it with a regex + * over the raw text, as this used to, left `/ip6/2001:0db8:0000:...:0001` and + * `/ip6/2001:db8::1` looking like two different addresses even though they are one, + * so a configuration edit could leave the old spelling behind in the autodial list. + */ +export function normalizeMultiaddrForCompare(s: string): string { + return canonicalMultiaddr(s); +} + +/** + * Whether two multiaddrs denote the same transport endpoint, ignoring a trailing + * `/p2p/` (a dial target usually carries it, `Connection.remoteAddr` may not). + * + * Compares the WHOLE remaining address, never a prefix: `/ip4/x/tcp/80` is a string + * prefix of `/ip4/x/tcp/8080`, so prefix matching would accept a connection on one + * port as proof for another — exactly the unverified-address case this is used to + * reject. + */ +export function isSameDialEndpoint(a: string, b: string): boolean { + const strip = (s: string): string => normalizeMultiaddrForCompare(s).replace(/\/p2p\/[^/]+$/, ''); + const left = strip(a); + return left.length > 0 && left === strip(b); +} + /** * Classify a libp2p dial error into a coarse status the UI can render distinctly. * diff --git a/backend/src/protocol/peer-announce.ts b/backend/src/protocol/peer-announce.ts index d43125dc9..1f7fb9799 100644 --- a/backend/src/protocol/peer-announce.ts +++ b/backend/src/protocol/peer-announce.ts @@ -1,7 +1,7 @@ import { trace } from '../logger.ts'; import { getLocalCidrs, shouldDenyDial } from './address-filter.ts'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; -import { peerIdFromString as peerIDFromString } from '@libp2p/peer-id'; +import { canonicalMultiaddr, extractDestinationPeerID } from './multiaddr-utils.ts'; import { LISH_TOPIC_PREFIX } from './constants.ts'; import { type Libp2p } from 'libp2p'; import { type BootstrapPeerOrigin } from '@shared'; @@ -47,6 +47,22 @@ const PEER_ANNOUNCE_MAX_ADDRS = 32; * latency cost (sub-2 announce cycles to fill peerStore). */ const PEER_ANNOUNCE_MAX_TOTAL_ADDRS = 128; +/** + * Hard bound on RAW entries examined per announce, before anything is deduplicated. + * + * {@link PEER_ANNOUNCE_MAX_TOTAL_ADDRS} caps the UNIQUE addresses admitted, which bounds + * what intake costs downstream but not what the walk itself costs: every raw entry is + * parsed, canonicalised and routability-tested first, and a message can carry thousands + * of duplicates or unparseable values that never reach the unique cap at all. Eight times + * the unique cap leaves ample room for a legitimate announce (which is already trimmed to + * the unique cap by its emitter) while putting a ceiling on the work one message can buy. + */ +const PEER_ANNOUNCE_MAX_RAW_ADDRS = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 8; +/** + * Longest announced address we will even attempt to parse. Real multiaddrs are well under + * this; anything longer is a parser workload, not an address. + */ +const PEER_ANNOUNCE_MAX_ADDR_LENGTH = 512; /** Max addrs we take from a single known peer when including transitive list. */ const PEER_ANNOUNCE_MAX_ADDRS_PER_PEER = 3; /** @@ -61,14 +77,98 @@ const PEER_ANNOUNCE_MAX_ADDRS_PER_PEER = 3; */ const PEER_ANNOUNCE_MEMBER_TTL_MS = PEER_ANNOUNCE_INTERVAL_SATURATED_MS * 3; +/** + * Per-source announce budget, in unique addresses admitted per minute. + * + * Every address that survives intake costs a dial, a status row and a snapshot, and + * nothing about gossipsub stops one topic subscriber from announcing as fast as it + * likes — so the cost of a single hostile (or merely broken) emitter is otherwise + * unbounded. The budget is spent per announcing peer ID, so throttling one source + * never starves the rest of the topic. + * + * Sizing: the worst LEGITIMATE emitter is a mid-convergence peer (peerStore 20..80, + * {@link PEER_ANNOUNCE_INTERVAL_STEADY_MS} cadence) sending the full + * {@link PEER_ANNOUNCE_MAX_TOTAL_ADDRS} list twice a minute — 256 addresses. The + * sustained rate matches that exactly, and the bucket holds one and a half cycles' + * worth so downward jitter, a topic re-join or a cold-start burst still pass intact. + * A source that exceeds it is not a shape we emit. + */ +const PEER_ANNOUNCE_RATE_PER_MINUTE = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 2; +/** Bucket depth — burst allowance on top of {@link PEER_ANNOUNCE_RATE_PER_MINUTE}. */ +const PEER_ANNOUNCE_RATE_BURST = PEER_ANNOUNCE_MAX_TOTAL_ADDRS * 3; +/** + * Cap on tracked sources. Buckets are keyed by peer ID, which is unbounded input, so + * the map is an LRU: the least recently heard-from source is evicted first. Eviction + * hands that source a fresh budget if it ever returns, which is acceptable — refilling + * the table costs an attacker a distinct peer ID per slot, and 1024 is far above any + * real topic's subscriber count. + */ +const PEER_ANNOUNCE_RATE_MAX_SOURCES = 1024; +/** Bucket key for an announce that arrived without an attributable sender. */ +const PEER_ANNOUNCE_UNKNOWN_SOURCE = ''; + +/** + * Token bucket keyed by announcing peer ID, bounding how many addresses one source + * can push through peer-announce intake per unit of time. + * + * Partial grants are deliberate: a source over budget is trimmed rather than silenced, + * so a legitimate peer that overshoots still makes discovery progress. `now` is a + * parameter rather than a read of the clock so the refill curve is testable. + */ +export class AnnounceRateLimiter { + private readonly buckets = new Map(); + private readonly burst: number; + private readonly perMinute: number; + private readonly maxSources: number; + + constructor(burst: number = PEER_ANNOUNCE_RATE_BURST, perMinute: number = PEER_ANNOUNCE_RATE_PER_MINUTE, maxSources: number = PEER_ANNOUNCE_RATE_MAX_SOURCES) { + this.burst = burst; + this.perMinute = perMinute; + this.maxSources = maxSources; + } + + /** + * Spend up to `wanted` tokens on behalf of `source` and return how many were + * granted (0..wanted). An unknown source starts with a full bucket. + */ + take(source: string, wanted: number, now: number = Date.now()): number { + if (wanted <= 0) return 0; + const existing = this.buckets.get(source); + let tokens = this.burst; + if (existing) { + // Re-insert below so Map iteration order stays least-recently-used first. + this.buckets.delete(source); + tokens = Math.min(this.burst, existing.tokens + (Math.max(0, now - existing.seenAt) * this.perMinute) / 60_000); + } + const granted = Math.min(wanted, Math.floor(tokens)); + this.buckets.set(source, { tokens: tokens - granted, seenAt: now }); + while (this.buckets.size > this.maxSources) { + const oldest = this.buckets.keys().next(); + if (oldest.done) break; + this.buckets.delete(oldest.value); + } + return granted; + } + + /** Forget every source's budget. Used when the owning manager is stopped. */ + clear(): void { + this.buckets.clear(); + } +} + /** Dependencies for PeerAnnounceManager. */ export interface PeerAnnounceManagerDeps { /** Returns the current libp2p node (may be null if not started or already stopped). */ getNode(): Libp2p | null; /** Returns the current pubsub instance (may be null). */ getPubsub(): any | null; - /** Broadcast a message on a gossipsub topic. */ - broadcast(topic: string, msg: Record): Promise; + /** + * Broadcast a message on a gossipsub topic, over the pubsub instance the caller + * captured — NOT over whatever pubsub the network happens to hold now. An emit spans + * one await per topic, so a restart in the middle of one would otherwise publish the + * finished run's payload onto the new node. + */ + broadcast(topic: string, msg: Record, pubsub: any): Promise; /** Process an inbound peer-announce payload: dial/tag discovered peers. */ addBootstrapPeers(multiaddrs: string[], networkID: string, origin: BootstrapPeerOrigin): Promise; } @@ -85,6 +185,16 @@ export class PeerAnnounceManager { private readonly deps: PeerAnnounceManagerDeps; private timer: NodeJS.Timeout | null = null; private stopped = false; + /** + * Which start() a tick belongs to, bumped by both start() and stop(). + * + * The `stopped` flag alone cannot tell a tick apart from its successor: a loop parked + * in `await peerStore.all()` across a stop() AND a start() found the flag false again + * and carried on, so two loops armed timers into one `timer` field and the next stop() + * cancelled only the last of them. A generation is not reusable, so the parked loop + * ends where it stands. + */ + private generation = 0; /** * Per-topic recently-seen subscribers (peerID → last-seen ms). Lets a * momentarily-disconnected same-network peer stay an eligible transitive-announce @@ -93,6 +203,8 @@ export class PeerAnnounceManager { * getSubscribers, so the cross-network leak stays closed. Pruned each emit(). */ private readonly topicMembers = new Map>(); + /** Per-announcing-peer intake budget — see {@link AnnounceRateLimiter}. */ + private readonly rateLimiter = new AnnounceRateLimiter(); constructor(deps: PeerAnnounceManagerDeps) { this.deps = deps; @@ -158,18 +270,32 @@ export class PeerAnnounceManager { /** Start the periodic emitter. Safe to call only once per start/stop cycle. */ start(): void { this.stopped = false; - this.scheduleNext().catch(() => { + const generation = ++this.generation; + this.scheduleNext(generation).catch(() => { /* first-tick scheduling failure would leave emitter stopped — acceptable fallback */ }); } - /** Stop the emitter. Idempotent. Any in-flight tick will not reschedule. */ + /** + * Stop the emitter and drop everything the previous run learned. Idempotent; any + * in-flight tick will not reschedule. + * + * Both maps are per-run state. Membership is read by leave-network to decide who to + * hang up, and it is recorded against the libp2p node that was running when the peer + * was seen — carrying it into the next start() means a leave acting on the previous + * node's mesh. The rate-limiter buckets are the same kind of thing from the other + * side: an exhausted budget surviving a restart throttles a source that has not sent + * anything to THIS run yet. + */ stop(): void { this.stopped = true; + this.generation++; if (this.timer) { clearTimeout(this.timer); this.timer = null; } + this.topicMembers.clear(); + this.rateLimiter.clear(); } /** Handle an inbound peer-announce pubsub message. */ @@ -184,59 +310,87 @@ export class PeerAnnounceManager { // do the same — every receiver must be defensive. const localCidrs = getLocalCidrs(); const rawCount = data.multiaddrs.length; - const filtered: string[] = []; + // Third stage: collapse addresses that mean the same thing. The cap counts + // UNIQUE addresses, so a message repeating one address 128 times no longer + // consumes the whole budget — and, more to the point, no longer turns into + // 128 markPending + dial + recordOutcome rounds downstream, since every stage + // after this one keys off the address string. Canonicalisation (not raw string + // equality) is what makes two spellings of one address — DNS case, expanded vs + // compressed IPv6 — count once. + const unique = new Map(); let droppedNonRoutable = 0; + let droppedDuplicate = 0; + let droppedAnonymous = 0; + let examined = 0; for (const a of data.multiaddrs) { - if (typeof a !== 'string' || a.length === 0) continue; - if (filtered.length >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; + // Counted over RAW entries, so duplicates and junk are spent from the same + // budget as anything else — the unique cap alone bounds only what survives. + if (++examined > PEER_ANNOUNCE_MAX_RAW_ADDRS) break; + if (typeof a !== 'string' || a.length === 0 || a.length > PEER_ANNOUNCE_MAX_ADDR_LENGTH) continue; + if (unique.size >= PEER_ANNOUNCE_MAX_TOTAL_ADDRS) break; try { - if (shouldDenyDial(Multiaddr(a), localCidrs)) { + const ma = Multiaddr(a); + if (shouldDenyDial(ma, localCidrs)) { droppedNonRoutable++; continue; } + // An address with no trailing /p2p/ names no identity, and every + // per-peer control downstream is keyed by one: backoff, quarantine, + // leave-network suppression and purge-by-peer-ID all silently do nothing + // for it, while a single successful dial is enough to park it on the + // recovery list that nothing can then take it off. Our own emitter always + // appends the identity, so requiring it costs nothing legitimate. + if (extractDestinationPeerID(ma) === null) { + droppedAnonymous++; + continue; + } } catch { // Unparseable multiaddr → drop (can't safely dial it anyway). droppedNonRoutable++; continue; } - filtered.push(a); + const canonical = canonicalMultiaddr(a); + if (unique.has(canonical)) { + droppedDuplicate++; + continue; + } + // Keep the spelling as announced: downstream keys status rows by this exact + // string, and rewriting it here would split one peer's row in two. + unique.set(canonical, a); + } + if (unique.size === 0) { + if (droppedNonRoutable > 0 || droppedDuplicate > 0 || droppedAnonymous > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${rawCount} addrs (${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, ${droppedAnonymous} without /p2p)`); + return; } - if (filtered.length === 0) { - if (droppedNonRoutable > 0) trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: dropped all ${droppedNonRoutable}/${rawCount} addrs as non-routable`); + // Rate-limit AFTER dedup: a duplicate flood must not be able to drain the + // sender's budget and starve the addresses it announced legitimately. + const source = fromPeerID ?? PEER_ANNOUNCE_UNKNOWN_SOURCE; + const admitted = this.rateLimiter.take(source, unique.size); + if (admitted === 0) { + trace(`[NET] peer-announce from ${source.slice(0, 16)}: rate limited, dropped all ${unique.size} addrs`); return; } - trace(`[NET] peer-announce from ${fromPeerID?.slice(0, 16) ?? 'unknown'}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, network ${networkID.slice(0, 8)})`); + const filtered = admitted < unique.size ? [...unique.values()].slice(0, admitted) : [...unique.values()]; + if (admitted < unique.size) trace(`[NET] peer-announce from ${source.slice(0, 16)}: rate limited to ${admitted}/${unique.size} addrs`); + trace(`[NET] peer-announce from ${source.slice(0, 16)}: ${filtered.length}/${rawCount} addrs (dropped ${droppedNonRoutable} non-routable, ${droppedDuplicate} duplicate, ${droppedAnonymous} without /p2p, network ${networkID.slice(0, 8)})`); // Pass networkID so per-peer outcomes from gossiped entries surface in the // UI under the network through which they arrived. Identity-mismatch // outcomes inside addBootstrapPeers also trigger purgeStalePeer. + // + // Deliberately NO peerStore.merge / keep-alive tagging here: an announce is + // an unverified claim. Writing tags for every mentioned peerID would (a) let + // any topic subscriber inject arbitrary peerIDs that we then persist and + // re-dial forever, and (b) refresh peerStore maxPeerAge for long-dead peers + // on every cycle, so they never expire. Keep-alive tagging happens only + // after a dial actually succeeds (addBootstrapPeers, peer:connect, re-dial + // maintenance) — a dead peer mentioned by gossip is still dialed below, it + // just no longer leaves a permanent peerStore footprint when unreachable. await this.deps.addBootstrapPeers(filtered, networkID, 'discovered'); - // Stamp `keep-alive-fleet` on every peer the announce mentioned. libp2p - // ReconnectQueue only acts on peers carrying a tag whose key starts with - // `keep-alive`; without this tag, fleet-discovered peers that drop are - // not re-dialed automatically. addBootstrapPeers() above tags via KEEP_ALIVE - // only for peer IDs it successfully extracts from multiaddrs — this adds - // the same treatment for every known peer, driving mesh maintenance. - const node = this.deps.getNode(); - if (node) { - for (const ma of filtered) { - try { - const mapath = Multiaddr(ma); - const pidComp = mapath.getComponents().find(c => c.code === 421); - const pid = pidComp?.value; - if (!pid) continue; - if (pid === node.peerId.toString()) continue; - await node.peerStore.merge(peerIDFromString(pid), { - tags: { 'keep-alive-fleet': { value: 50 } }, - }); - } catch { - /* invalid multiaddr — skip */ - } - } - } } - private async scheduleNext(): Promise { - if (this.stopped) return; + /** Arm the next announce tick for `generation`; see {@link generation}. */ + private async scheduleNext(generation: number): Promise { + if (this.isSuperseded(generation)) return; const node = this.deps.getNode(); const pubsub = this.deps.getPubsub(); if (!node || !pubsub) return; @@ -250,26 +404,33 @@ export class PeerAnnounceManager { } catch { base = PEER_ANNOUNCE_INTERVAL_STEADY_MS; } - if (this.stopped) return; + // Checked before the timer is armed, not only before the work: a superseded loop + // that still armed one would leave two timers behind a single `timer` field. + if (this.isSuperseded(generation)) return; const jitter = Math.floor((Math.random() * 2 - 1) * base * PEER_ANNOUNCE_JITTER_RATIO); const delay = Math.max(5_000, base + jitter); this.timer = setTimeout(async () => { // Guard: stop() may have been called while we were sleeping. - if (this.stopped) return; + if (this.isSuperseded(generation)) return; try { - await this.emit(); + await this.emit(generation); } catch (err: any) { trace(`[NET] peer-announce emit error: ${err?.message ?? err}`); } // Guard again before scheduling the next tick. - if (this.stopped) return; - this.scheduleNext().catch(() => { + if (this.isSuperseded(generation)) return; + this.scheduleNext(generation).catch(() => { /* schedule is async but errors handled inline */ }); }, delay); } - private async emit(): Promise { + /** True once this tick's start() has been superseded by a stop() or a newer start(). */ + private isSuperseded(generation: number): boolean { + return this.stopped || generation !== this.generation; + } + + private async emit(generation: number = this.generation): Promise { const node = this.deps.getNode(); const pubsub = this.deps.getPubsub(); if (!node || !pubsub) return; @@ -279,6 +440,10 @@ export class PeerAnnounceManager { // threshold below. Broadcasting is what the threshold gates, not tracking. this.refreshMembers(lishTopics, pubsub); const allPeers = await node.peerStore.all(); + // The node and pubsub captured above belong to the run this tick started in; a + // restart landing in that await would otherwise have us publish the previous run's + // addresses onto the new node's topics. + if (this.isSuperseded(generation)) return; if (allPeers.length < PEER_ANNOUNCE_MIN_PEER_STORE) return; if (lishTopics.length === 0) return; const localCidrs = getLocalCidrs(); @@ -312,6 +477,10 @@ export class PeerAnnounceManager { // the rest of their OWN network in one hop, without cross-network leak. let skippedTransitive = 0; for (const topic of lishTopics) { + // Once per iteration, not once before the loop: every iteration below ends in an + // awaited publish, and a stop/start landing in any of them makes each remaining + // topic a topic of a node this emit knows nothing about. + if (this.isSuperseded(generation)) return; const current = new Set(); try { for (const p of pubsub.getSubscribers(topic)) current.add(p.toString()); @@ -337,7 +506,17 @@ export class PeerAnnounceManager { skippedTransitive++; continue; } - const full = base.includes('/p2p/') ? base : `${base}/p2p/${pid}`; + // "Contains a /p2p/" is not the same question as "ends at THIS peer". A + // stale or poisoned peerStore address of A that terminates in /p2p/B was + // treated as already identified and broadcast verbatim, so every receiver + // learned it as B's address and dialed the wrong identity. Ask the + // destination — the same way the dial paths do — and skip what disagrees. + const destination = extractDestinationPeerID(addr.multiaddr); + if (destination !== null && destination !== pid) { + trace(`[NET] peer-announce skipping addr of ${pid.slice(0, 16)} that resolves to ${destination.slice(0, 16)}: ${base}`); + continue; + } + const full = destination === pid ? base : `${base}/p2p/${pid}`; if (!collected.has(full)) transitiveAdded++; collected.add(full); perPeer++; @@ -350,10 +529,15 @@ export class PeerAnnounceManager { const msg: PeerAnnounceMessage = { type: 'peer-announce', multiaddrs: Array.from(collected) }; trace(`[NET] peer-announce emit topic=${topic.slice(0, 16)}: ${collected.size} addrs (self + ${transitiveAdded} scoped transitive)`); try { - await this.deps.broadcast(topic, msg as unknown as Record); + // Bound to the pubsub captured at the top of this emit, so even a publish that + // slips past the checks cannot reach the successor run's transport. + await this.deps.broadcast(topic, msg as unknown as Record, pubsub); } catch (err: any) { trace(`[NET] peer-announce publish failed topic=${topic}: ${err?.message ?? err}`); } + // The publish above is the await this loop exists to protect: a run that ended + // while it was outstanding must stop here rather than roll on to the next topic. + if (this.isSuperseded(generation)) return; } if (skippedSelf > 0 || skippedTransitive > 0) { trace(`[NET] peer-announce filter: skipped ${skippedSelf} self + ${skippedTransitive} transitive non-routable addrs`); diff --git a/backend/tests/unit/api/factory-reset-orchestrator.test.ts b/backend/tests/unit/api/factory-reset-orchestrator.test.ts index ff84c2554..af9ef55b9 100644 --- a/backend/tests/unit/api/factory-reset-orchestrator.test.ts +++ b/backend/tests/unit/api/factory-reset-orchestrator.test.ts @@ -229,7 +229,7 @@ describe('buildFactoryResetHandler — partial failure', () => { expect(res.results.every(r => r.ok)).toBe(true); }); - it('prepare failure is best-effort — wipes still run and success reflects only categories', async () => { + it('prepare failure is a barrier — the download wipe is skipped and success is false', async () => { const ran: string[] = []; const deps = makeDeps({ stopVerifyAll: async () => { @@ -244,10 +244,48 @@ describe('buildFactoryResetHandler — partial failure', () => { }); const handler = buildFactoryResetHandler(deps); const res = await handler({ downloads: true, settings: false, identity: false, networks: false, peers: false }); - // prepare threw but downloads still ran. - expect(ran).toContain('stopVerify'); - expect(ran).toContain('downloads'); - expect(res.success).toBe(true); + // Live transfers were never stopped, so wiping the tables they write to is not safe. + expect(ran).toEqual(['stopVerify']); + expect(res.success).toBe(false); + expect(res.phases[0]).toEqual({ phase: 'prepare', ok: false, detail: 'verify-stop boom' }); + }); + + it('a node that cannot be stopped blocks every destructive wipe and the restart', async () => { + const called: string[] = []; + const deps = makeDeps({ + networkOverride: { + stopAllNetworks: () => Promise.reject(new Error('node.stop failed')), + startEnabledNetworks: () => { + called.push('restart'); + return Promise.resolve(); + }, + clearDatastore: () => { + called.push('clearDatastore'); + return Promise.resolve(); + }, + clearPeerstore: () => { + called.push('clearPeerstore'); + return Promise.resolve(); + }, + }, + dataServerOverride: { + clearLishs: () => { + called.push('clearLishs'); + }, + clearLishnets: () => { + called.push('clearLishnets'); + }, + }, + }); + const handler = buildFactoryResetHandler(deps); + const res = await handler({ settings: false, identity: true, downloads: true, networks: true, peers: true }); + + // The node may still own its datastore, its peerstore and its identity. Wiping any + // of them here — and then bringing a second node up over the result — is exactly + // what the barrier exists to stop. + expect(called).toEqual([]); + expect(res.success).toBe(false); + expect(res.results.every(r => !r.ok)).toBe(true); }); }); diff --git a/backend/tests/unit/api/factory-reset.test.ts b/backend/tests/unit/api/factory-reset.test.ts index 6b5257735..2421d3865 100644 --- a/backend/tests/unit/api/factory-reset.test.ts +++ b/backend/tests/unit/api/factory-reset.test.ts @@ -5,6 +5,7 @@ describe('runFactoryReset', () => { it('runs EVERY selected category even when some fail — no category is skipped because a previous one threw', async () => { const ran: string[] = []; const res = await runFactoryReset({ + prepare: (): void => {}, downloads: (): void => { ran.push('downloads'); }, @@ -33,34 +34,128 @@ describe('runFactoryReset', () => { }); it('only reports selected categories', async () => { - const res = await runFactoryReset({ downloads: (): void => {} }); + const res = await runFactoryReset({ prepare: (): void => {}, downloads: (): void => {} }); expect(res.results).toEqual([{ category: 'downloads', ok: true }]); expect(res.success).toBe(true); }); - it('runs prepare before the wipes and restart after — both best-effort (their failure neither aborts nor flips success)', async () => { + /** + * `prepare` is a barrier, so its ABSENCE proves exactly as little as its failure. Treating + * a missing barrier as a passed one let this wipe the downloads, the networks, the + * peerstore and the identity out from under a running node and report success. + */ + it('a destructive category selected without any prepare is skipped, not run', async () => { + const ran: string[] = []; + const res = await runFactoryReset({ + downloads: (): void => { + ran.push('downloads'); + }, + networks: (): void => { + ran.push('networks'); + }, + peers: (): void => { + ran.push('peers'); + }, + identity: (): void => { + ran.push('identity'); + }, + settings: (): void => { + ran.push('settings'); + }, + restart: (): void => { + ran.push('restart'); + }, + }); + + expect(ran).toEqual(['settings']); + expect(res.success).toBe(false); + expect(res.results.filter(r => !r.ok).map(r => r.category)).toEqual(['downloads', 'networks', 'peers', 'identity']); + expect(res.phases[0]).toEqual({ phase: 'prepare', ok: false, detail: 'no prepare step was supplied, so nothing was stopped' }); + expect(res.phases[1]?.ok).toBe(false); + }); + + it('a settings-only reset needs no barrier and reports no prepare phase', async () => { + const res = await runFactoryReset({ settings: (): void => {} }); + expect(res.success).toBe(true); + expect(res.phases).toEqual([]); + expect(res.results).toEqual([{ category: 'settings', ok: true }]); + }); + + it('runs prepare before the wipes and restart after, reporting both as phases', async () => { const order: string[] = []; const res = await runFactoryReset({ prepare: (): void => { order.push('prepare'); - throw new Error('prep boom'); }, downloads: (): void => { order.push('downloads'); }, restart: (): void => { order.push('restart'); - throw new Error('restart boom'); }, }); expect(order).toEqual(['prepare', 'downloads', 'restart']); - // prepare/restart failures are infrastructure — not in results, don't fail the reset. expect(res.success).toBe(true); + expect(res.phases).toEqual([ + { phase: 'prepare', ok: true }, + { phase: 'restart', ok: true }, + ]); + }); + + it('a failed prepare skips every wipe that needs the node down, and the restart', async () => { + const ran: string[] = []; + const res = await runFactoryReset({ + prepare: (): void => { + throw new Error('stopAllNetworks boom'); + }, + downloads: (): void => { + ran.push('downloads'); + }, + networks: (): void => { + ran.push('networks'); + }, + peers: (): void => { + ran.push('peers'); + }, + identity: (): void => { + ran.push('identity'); + }, + // The one wipe that touches neither the node nor the transfers. + settings: (): void => { + ran.push('settings'); + }, + restart: (): void => { + ran.push('restart'); + }, + }); + + expect(ran).toEqual(['settings']); + expect(res.success).toBe(false); + expect(res.results.filter(r => !r.ok).map(r => r.category)).toEqual(['downloads', 'networks', 'peers', 'identity']); + expect(res.results.find(r => r.category === 'settings')).toEqual({ category: 'settings', ok: true }); + expect(res.phases[0]).toEqual({ phase: 'prepare', ok: false, detail: 'stopAllNetworks boom' }); + expect(res.phases[1]?.phase).toBe('restart'); + expect(res.phases[1]?.ok).toBe(false); + }); + + it('a failed restart forces success=false even when every category passed', async () => { + const res = await runFactoryReset({ + prepare: (): void => {}, + downloads: (): void => {}, + restart: (): void => { + throw new Error('restart boom'); + }, + }); expect(res.results).toEqual([{ category: 'downloads', ok: true }]); + expect(res.success).toBe(false); + expect(res.phases).toEqual([ + { phase: 'prepare', ok: true }, + { phase: 'restart', ok: false, detail: 'restart boom' }, + ]); }); it('reports success=true when every selected category passes', async () => { - const res = await runFactoryReset({ settings: (): void => {}, identity: (): void => {}, downloads: (): void => {}, networks: (): void => {} }); + const res = await runFactoryReset({ prepare: (): void => {}, settings: (): void => {}, identity: (): void => {}, downloads: (): void => {}, networks: (): void => {} }); expect(res.success).toBe(true); expect(res.results.map(r => r.category)).toEqual(['downloads', 'networks', 'identity', 'settings']); expect(res.results.every(r => r.ok)).toBe(true); diff --git a/backend/tests/unit/lishnet/enable-serialisation.test.ts b/backend/tests/unit/lishnet/enable-serialisation.test.ts new file mode 100644 index 000000000..034d4c46b --- /dev/null +++ b/backend/tests/unit/lishnet/enable-serialisation.test.ts @@ -0,0 +1,580 @@ +import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; +import { initLISHnetsTables, addLISHnet, getLISHnet, setLISHnetEnabled } from '../../../src/db/lishnets.ts'; +import { Networks } from '../../../src/lishnet/lishnets.ts'; + +/** + * Enable and disable of one lishnet must produce the state the LAST request asked + * for — in the database, in the pubsub subscription and in the callbacks the + * transfer layer listens to. + * + * Neither operation used to be serialised, and both await for a long time: a join + * waits on bootstrap dials, a leave disconnects peers one at a time. So an enable + * could announce a join after a disable had already left, and a leave could keep + * disconnecting the peers of a network that had just been re-enabled. + * + * Each request now runs to completion and then the next one converges on the row it + * left behind, so a contested toggle costs one redundant pass and reports honestly + * what happened. Abandoning the loser mid-flight instead is what left a cleanup + * half-done with a successor that had nothing left to finish. + */ + +const NET = 'net-a'; +const NAMED = { networkID: 'net-a', name: 'A' }; +const BOOTSTRAP = '/ip4/192.0.2.1/tcp/9090/p2p/12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +function makeMockNet() { + return { + subscribed: [] as string[], + unsubscribed: [] as string[], + disconnected: [] as string[], + /** Set to hold the next bootstrap dial / peer disconnect open. */ + dialGate: null as null | Promise, + disconnectGate: null as null | Promise, + topicPeers: new Map(), + getTopicPeers(id: string): string[] { + return this.topicPeers.get(id) ?? []; + }, + getRecentTopicMembers(): string[] { + return []; + }, + isRunning(): boolean { + return true; + }, + subscribeTopic(id: string): boolean { + this.subscribed.push(id); + return true; + }, + unsubscribeTopic(id: string): void { + this.unsubscribed.push(id); + this.topicPeers.delete(id); + }, + isBootstrapOrRelayPeer(): boolean { + return false; + }, + async disconnectPeer(pid: string): Promise { + if (this.disconnectGate) await this.disconnectGate; + this.disconnected.push(pid); + }, + pruneConfiguredBootstrapPeer(): void {}, + resetBootstrapStatus(): void {}, + pruneBootstrapAddresses(): void {}, + pruneBootstrapStatus(): void {}, + clearRedialSuppressionForNetwork(): void {}, + async addBootstrapPeers(): Promise { + if (this.dialGate) await this.dialGate; + }, + }; +} + +function makeNetworks(net: ReturnType, db: Database, joined: string[]) { + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).db = db; + (networks as any).network = net; + (networks as any).joinedNetworks = new Set(joined); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + (networks as any).announcedJoined = new Map(joined.map(id => [id, true])); + const events: string[] = []; + (networks as any)._onNetworkJoined = (id: string): void => { + events.push(`joined:${id}`); + }; + (networks as any)._onNetworkLeft = (id: string): void => { + events.push(`left:${id}`); + }; + return { networks, events }; +} + +describe('Networks.setEnabled — serialised per lishnet', () => { + let db: Database; + let net: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: false, created: new Date().toISOString() }); + net = makeMockNet(); + }); + + it('an enable overtaken by a disable ends disabled, both transitions announced', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks, events } = makeNetworks(net, db, []); + + const enabling = networks.setEnabled(NET, true); + // Enough turns for the enable to take its locks, subscribe, and park on the dial — + // the disable has to arrive with the join already half-done for this to mean + // anything. + for (let i = 0; i < 10; i++) await Promise.resolve(); + // The user changes their mind while the bootstrap dial is still outstanding. + const disabling = networks.setEnabled(NET, false); + gate.resolve(); + await Promise.all([enabling, disabling]); + + // The join really did happen — it subscribed the topic before it parked — so saying + // so and then saying it was undone is the honest report. Cancelling it half-way is + // what left the subscription and the dials behind with nobody to clean them up. + expect(events).toEqual([`joined:${NET}`, `left:${NET}`]); + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + expect(net.unsubscribed).toEqual([NET]); + }); + + it('a disable overtaken by an enable finishes its cleanup before the rejoin', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks, events } = makeNetworks(net, db, [NET]); + + const disabling = networks.setEnabled(NET, false); + // Enough turns for the leave to be genuinely mid-cleanup, parked on a peer disconnect, + // when the re-enable arrives. + for (let i = 0; i < 10; i++) await Promise.resolve(); + const enabling = networks.setEnabled(NET, true); + gate.resolve(); + await Promise.all([disabling, enabling]); + + expect(events).toEqual([`left:${NET}`, `joined:${NET}`]); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + // The peer the leave was disconnecting when the re-enable arrived is disconnected, + // not stranded: abandoning the loop there left the tag, the peerStore record and the + // connection installed with the successor believing the leave was already done. + expect(net.disconnected).toContain('p-only-a'); + }); + + it('three fast toggles land on the state the last one asked for', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks, events } = makeNetworks(net, db, []); + + const first = networks.setEnabled(NET, true); + await Promise.resolve(); + const second = networks.setEnabled(NET, false); + const third = networks.setEnabled(NET, true); + gate.resolve(); + await Promise.all([first, second, third]); + + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + // Three toggles still cost one join. The database phases are short and run first, so + // by the time the second and third reconciles get the lock the row already holds the + // final value and there is nothing left for them to change — the collapse falls out + // of converging on the row rather than out of cancelling anybody. + expect(events).toEqual([`joined:${NET}`]); + expect(net.unsubscribed).toEqual([]); + }); + + /** + * Two identical disables. The first used to bail out of its peer-disconnect loop the + * moment the second merely ARRIVED, and the second then found the network already + * unsubscribed and returned at once — so the peers of a network nobody was in kept + * their connections, and nothing was ever going to come back for them. + */ + it('a second identical disable does not strand the first one’s peer cleanup', async () => { + net.topicPeers.set(NET, ['p-one', 'p-two', 'p-three']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks } = makeNetworks(net, db, [NET]); + + const first = networks.setEnabled(NET, false); + for (let i = 0; i < 10; i++) await Promise.resolve(); + const second = networks.setEnabled(NET, false); + gate.resolve(); + await Promise.all([first, second]); + + for (const pid of ['p-one', 'p-two', 'p-three']) expect(net.disconnected).toContain(pid); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + /** + * The transfer-layer observers iterate downloaders and mutate them, and one that throws + * used to come back out as a failed RPC — with the row written, the topic subscribed and + * `announcedJoined` already holding the new state. Retrying found nothing left to + * announce, so the event never reached the client for a network that really had joined. + */ + it('an observer that throws does not fail the transition it was told about', async () => { + const { networks } = makeNetworks(net, db, []); + (networks as any)._onNetworkJoined = (): void => { + throw new Error('downloader blew up'); + }; + + const result = await networks.setEnabled(NET, true); + + expect(result).toEqual({ found: true, transitioned: true, joined: true, network: NAMED }); + expect(net.subscribed).toEqual([NET]); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + }); + + it('an uncontested enable still joins and announces it', async () => { + const { networks, events } = makeNetworks(net, db, []); + + await networks.setEnabled(NET, true); + + expect(events).toEqual(['joined:' + NET]); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect(net.subscribed).toEqual([NET]); + }); + + it('an uncontested disable still leaves and announces it', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const { networks, events } = makeNetworks(net, db, [NET]); + + await networks.setEnabled(NET, false); + + expect(events).toEqual(['left:' + NET]); + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect(net.disconnected).toContain('p-only-a'); + }); +}); + +/** + * A delete is a row write AND a runtime change, and the two used to be separate lock + * acquisitions with an abortable leave in between. An enable arriving in that window + * superseded the leave, rejoined the topic, and then watched the delete remove the row: + * subscribed to a lishnet the database has never heard of. + */ +describe('Networks.delete — terminal against a concurrent enable', () => { + let db: Database; + let net: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: true, created: new Date().toISOString() }); + net = makeMockNet(); + }); + + /** Let the parked operation actually reach its gate before the racer arrives. */ + async function settle(): Promise { + for (let i = 0; i < 10; i++) await Promise.resolve(); + } + + it('an enable that lands mid-delete gets "not found" instead of rejoining', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks } = makeNetworks(net, db, [NET]); + + const deleting = networks.delete(NET); + await settle(); + const enabling = networks.setEnabled(NET, true); + gate.resolve(); + const [deleted, enabled] = await Promise.all([deleting, enabling]); + + expect(deleted).toBe(true); + expect(enabled).toEqual({ found: false, transitioned: false, joined: false }); + expect(getLISHnet(db, NET)).toBeUndefined(); + // The three things that must agree: no row, not joined, not subscribed. + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + expect(net.subscribed).toEqual([]); + expect(net.unsubscribed).toEqual([NET]); + }); + + /** + * Both maps are keyed by arbitrary network IDs and nothing used to remove an entry, so a + * node that creates and deletes networks over a long uptime grew them for ever. + */ + it('a deleted lishnet leaves no per-lishnet state behind', async () => { + const { networks } = makeNetworks(net, db, [NET]); + + await networks.delete(NET); + + expect((networks as any).networkOperations.size).toBe(0); + expect((networks as any).announcedJoined.size).toBe(0); + }); + + it('keeps the lock of a deleted lishnet while something is still queued on it', async () => { + const { networks } = makeNetworks(net, db, [NET]); + const lock = (networks as any).operationLock(NET); + const release = await lock.acquire(); + + const deleting = networks.delete(NET); + await settle(); + // The delete's own reconcile is queued on the lock held here. Dropping the mutex out + // of the map now would give the next caller a second, independent one for this + // lishnet — two locks guarding one transition, which is worse than the leak. + expect((networks as any).networkOperations.get(NET)).toBe(lock); + + release(); + await deleting; + expect((networks as any).networkOperations.size).toBe(0); + }); + + it('deleting an unknown lishnet reports it rather than pretending', async () => { + const { networks } = makeNetworks(net, db, []); + expect(await networks.delete('no-such-net')).toBe(false); + }); +}); + +/** + * The API turns this result into a `lishnets:joined` / `lishnets:left` broadcast, so it + * has to say what actually happened. A bare "the network exists" made an overruled + * request and an idempotent one both look like a settled transition, and the client was + * told the network had joined when it had just been disabled. + */ +describe('Networks.setEnabled — what the result claims', () => { + let db: Database; + let net: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: true, created: new Date().toISOString() }); + net = makeMockNet(); + }); + + it('a request whose state was already settled by then reports no transition', async () => { + net.topicPeers.set(NET, ['p-only-a']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks } = makeNetworks(net, db, [NET]); + + const holding = networks.setEnabled(NET, false); + for (let i = 0; i < 10; i++) await Promise.resolve(); + const older = networks.setEnabled(NET, true); + const newer = networks.setEnabled(NET, false); + gate.resolve(); + const [holdingResult, olderResult, newerResult] = await Promise.all([holding, older, newer]); + + // The enable in the middle is the interesting one: the disable after it wrote the row + // before either reconcile ran, so by the enable's turn the desired state was already + // "disabled" and it had nothing to apply. `transitioned: false` with `joined: false` + // is what the API needs to hear — it must not broadcast a join that did not happen. + expect(holdingResult).toEqual({ found: true, transitioned: true, joined: false, network: NAMED }); + expect(olderResult).toEqual({ found: true, transitioned: false, joined: false, network: NAMED }); + expect(newerResult).toEqual({ found: true, transitioned: false, joined: false, network: NAMED }); + expect(getLISHnet(db, NET)!.enabled).toBe(false); + }); + + it('an enable of an already-joined network reports no transition', async () => { + const { networks } = makeNetworks(net, db, [NET]); + + expect(await networks.setEnabled(NET, true)).toEqual({ found: true, transitioned: false, joined: true, network: NAMED }); + }); + + /** + * The API broadcasts `lishnets:joined` / `lishnets:left` from this result. It used to + * read the row itself before awaiting the call, which raced the catalog both ways: a + * network still being added read as undefined and its join was never broadcast at all, + * and a rename queued ahead of the enable made the event carry the previous name. + */ + it('names the row the enable settled, not one a queued rename has replaced', async () => { + setLISHnetEnabled(db, NET, false); + const { networks } = makeNetworks(net, db, []); + const release = await (networks as any).catalogMutex.acquire(); + + const renaming = networks.update({ networkID: NET, name: 'renamed', description: '', bootstrapPeers: [BOOTSTRAP], enabled: false, created: new Date().toISOString() }); + const enabling = networks.setEnabled(NET, true); + release(); + const [, result] = await Promise.all([renaming, enabling]); + + // Whichever of the two reconciles reaches the lock first settles the join; what this + // pins down is the name — it is the one this call's own critical section wrote, never + // a value read before or after the await. + expect(result.joined).toBe(true); + expect(result.network).toEqual({ networkID: NET, name: 'renamed' }); + }); + + it('names a network that only came into existence while it waited', async () => { + const { networks } = makeNetworks(net, db, []); + const release = await (networks as any).catalogMutex.acquire(); + + const adding = networks.addIfNotExists({ networkID: 'net-new', name: 'New', description: '', bootstrapPeers: [], created: new Date().toISOString() }); + const enabling = networks.setEnabled('net-new', true); + release(); + const [, result] = await Promise.all([adding, enabling]); + + expect(result).toEqual({ found: true, transitioned: true, joined: true, network: { networkID: 'net-new', name: 'New' } }); + }); + + it('a real enable reports the transition it settled', async () => { + setLISHnetEnabled(db, NET, false); + const { networks } = makeNetworks(net, db, []); + + expect(await networks.setEnabled(NET, true)).toEqual({ found: true, transitioned: true, joined: true, network: NAMED }); + }); +}); + +/** + * replace() decides which per-ID locks it needs from a snapshot of the ID set, so an ID + * that comes into existence while it waits was reconciled — and could be deleted from the + * database — with nobody holding its lock, right through the add that was still joining + * it. The catalog mutex is what stops the set from moving between the snapshot and the + * locks. + */ +describe('Networks — operations that change the set of lishnets', () => { + const NET_B = 'net-b'; + let db: Database; + let net: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: false, created: new Date().toISOString() }); + net = makeMockNet(); + }); + + async function settle(): Promise { + for (let i = 0; i < 10; i++) await Promise.resolve(); + } + + /** + * The user's last request has to win, whichever API it came through. Every public writer + * queues on the catalog mutex before it awaits anything else, and the mutex dispatches + * first come, first served, so the last request also writes its row last and the + * reconcile that follows converges on it. + */ + function rowOf(id: string) { + return { networkID: id, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: false, created: new Date().toISOString() }; + } + + it('a setEnabled issued after an update wins over it', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + // Something holds net-a's lock, so both writers below queue behind it. + const holdingA = networks.setEnabled(NET, true); + await settle(); + const updating = networks.update({ ...rowOf(NET), name: 'renamed', enabled: false }); + const enabling = networks.setEnabled(NET, true); + + gate.resolve(); + await Promise.all([holdingA, updating, enabling]); + + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + // The edit's own field change is not the newer request's to discard. + expect(getLISHnet(db, NET)!.name).toBe('renamed'); + }); + + it('an update issued after a setEnabled wins over it', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + const holdingA = networks.setEnabled(NET, true); + await settle(); + const enabling = networks.setEnabled(NET, true); + const updating = networks.update({ ...rowOf(NET), enabled: false }); + + gate.resolve(); + await Promise.all([holdingA, enabling, updating]); + + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('a setEnabled issued after an import wins over it', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + const holdingA = networks.setEnabled(NET, true); + await settle(); + const importing = networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], created: new Date().toISOString() } as any, false); + const enabling = networks.setEnabled(NET, true); + + gate.resolve(); + await Promise.all([holdingA, importing, enabling]); + + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + }); + + it('a setEnabled issued after a replace wins over it', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + const holdingA = networks.setEnabled(NET, true); + await settle(); + const replacing = networks.replace([rowOf(NET)]); + const enabling = networks.setEnabled(NET, true); + + gate.resolve(); + await Promise.all([holdingA, replacing, enabling]); + + expect(getLISHnet(db, NET)!.enabled).toBe(true); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + }); + + it('a network dropped by replace is cleaned up terminally, like a delete', async () => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: true, created: new Date().toISOString() }); + net.topicPeers.set(NET, ['p-only-a']); + const gate = deferred(); + net.disconnectGate = gate.promise; + const { networks } = makeNetworks(net, db, [NET]); + + const replacing = networks.replace([]); + await settle(); + // The enable's row is already gone, so it answers "not found" and reconciles nothing. + // If the leave it arrived during were abortable, nothing would ever finish this + // cleanup: no row, no subscription, and the peers still connected. + const enabling = networks.setEnabled(NET, true); + gate.resolve(); + await Promise.all([replacing, enabling]); + + expect(getLISHnet(db, NET)).toBeUndefined(); + expect(net.disconnected).toContain('p-only-a'); + }); + + /** + * The global catalog lock used to be held for the whole of a join, bootstrap dials and + * all — seconds per address, sequentially. Every unrelated lishnet's edit, add, delete + * and import queued behind whichever single network happened to be dialing, and so did + * the shutdown and the factory reset, which presented to the user as a frozen app. + */ + it('a slow join of one lishnet does not block writes to another', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + const joining = networks.setEnabled(NET, true); + await settle(); + + // net-b shares nothing with net-a — no bootstrap peers of its own, so it has no dial + // to wait on — and neither of these may wait on net-a's. + expect(await networks.add({ ...rowOf(NET_B), name: 'B', bootstrapPeers: [] })).toBe(true); + expect(await networks.setEnabled(NET_B, true)).toEqual({ found: true, transitioned: true, joined: true, network: { networkID: NET_B, name: 'B' } }); + expect(getLISHnet(db, NET)!.enabled).toBe(true); + + gate.resolve(); + await joining; + }); + + it('a network added while replace waits is not wiped out behind its back', async () => { + const gate = deferred(); + net.dialGate = gate.promise; + const { networks } = makeNetworks(net, db, []); + + // Something is holding net-a's lock, so replace() parks with a snapshot of the ID + // set that predates net-b entirely. + const holdingA = networks.setEnabled(NET, true); + await settle(); + const replacing = networks.replace([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [BOOTSTRAP], enabled: true, created: new Date().toISOString() }]); + await settle(); + const adding = networks.add({ networkID: NET_B, name: 'B', description: '', bootstrapPeers: [BOOTSTRAP], enabled: true, created: new Date().toISOString() }); + await settle(); + + gate.resolve(); + await Promise.all([holdingA, replacing, adding]); + + // Without the catalog mutex, replace's rewrite of the list dropped net-b's row while + // the add was still joining it: subscribed, in joinedNetworks, and no row at all. + expect(getLISHnet(db, NET_B)).toBeDefined(); + expect((networks as any).joinedNetworks.has(NET_B)).toBe(true); + }); +}); diff --git a/backend/tests/unit/lishnet/import-reconcile.test.ts b/backend/tests/unit/lishnet/import-reconcile.test.ts new file mode 100644 index 000000000..07ba26bbd --- /dev/null +++ b/backend/tests/unit/lishnet/import-reconcile.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; +import { initLISHnetsTables, addLISHnet, getLISHnet } from '../../../src/db/lishnets.ts'; +import { Networks } from '../../../src/lishnet/lishnets.ts'; + +/** + * Importing a network writes the database. It must also reach the running node: + * importing an already-joined network used to leave the live bootstrap list, statuses + * and autodial addresses on the previous configuration, and importing an active + * network as disabled left it joined until the next restart. + */ + +const NET = 'net-a'; +const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; +const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; +const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + +function makeMockNet() { + return { + subscribed: [] as string[], + unsubscribed: [] as string[], + dialledLists: [] as Array<{ networkID: string; peers: string[]; origin: string }>, + prunedStatus: [] as Array<{ networkID: string; keep: string[] }>, + prunedAddresses: [] as string[][], + prunedBootstrap: [] as string[], + getTopicPeers: (): string[] => [], + getRecentTopicMembers: (): string[] => [], + isBootstrapOrRelayPeer: (): boolean => false, + async disconnectPeer(): Promise {}, + pruneConfiguredBootstrapPeer(pid: string): void { + this.prunedBootstrap.push(pid); + }, + resetBootstrapStatus(): void {}, + pruneBootstrapAddresses(addresses: string[]): void { + this.prunedAddresses.push(addresses); + }, + pruneBootstrapStatus(networkID: string, keep: string[]): void { + this.prunedStatus.push({ networkID, keep }); + }, + clearRedialSuppressionForNetwork(): void {}, + isRunning(): boolean { + return true; + }, + subscribeTopic(id: string): boolean { + this.subscribed.push(id); + return true; + }, + unsubscribeTopic(id: string): void { + this.unsubscribed.push(id); + }, + async addBootstrapPeers(peers: string[], networkID: string, origin: string): Promise { + this.dialledLists.push({ networkID, peers, origin }); + }, + }; +} + +function bare(db: Database, mock: ReturnType, joined: string[]) { + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).db = db; + (networks as any).network = mock; + (networks as any).joinedNetworks = new Set(joined); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + (networks as any).announcedJoined = new Map(joined.map(id => [id, true])); + return networks; +} + +describe('Networks.importFromLISHnet — the runtime follows the import', () => { + let db: Database; + let mock: ReturnType; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + mock = makeMockNet(); + }); + + it('switches a joined network over to the imported bootstrap list', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, [NET]); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_B] } as any, true); + + expect(getLISHnet(db, NET)!.bootstrapPeers).toEqual([ADDR_B]); + expect(mock.prunedStatus).toEqual([{ networkID: NET, keep: [ADDR_B] }]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_B], origin: 'configured' }]); + }); + + it('leaves an active network imported as disabled', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, [NET]); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, false); + + expect(getLISHnet(db, NET)!.enabled).toBe(false); + expect(mock.unsubscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('joins a brand-new network imported as enabled', async () => { + const networks = bare(db, mock, []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, true); + + expect(mock.subscribed).toEqual([NET]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_A], origin: 'configured' }]); + }); + + it('touches nothing at runtime for a disabled network imported as disabled', async () => { + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: false, created: '2026-01-01T00:00:00.000Z' }); + const networks = bare(db, mock, []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A] } as any, false); + + expect(mock.subscribed).toEqual([]); + expect(mock.unsubscribed).toEqual([]); + expect(mock.dialledLists).toEqual([]); + }); +}); + +describe('Networks.replace — a wholesale rewrite reaches the runtime', () => { + it('leaves a joined network the rewrite dropped', async () => { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = bare(db, mock, [NET]); + + await networks.replace([]); + + expect(mock.unsubscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('keeps a re-listed network joined', async () => { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = bare(db, mock, [NET]); + + await networks.replace([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], enabled: true, created: '2026-01-01T00:00:00.000Z' }]); + + expect(mock.unsubscribed).toEqual([]); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + }); +}); + +/** + * Normalisation belongs to the write, not to the caller. Only the two edit paths used to + * apply it, so import, add and wholesale replace could store whitespace-padded values + * that fail to parse at dial time, and two spellings of one endpoint that each earn their + * own probe and their own status row. + */ +describe('bootstrap lists are normalised by whoever writes them', () => { + const PADDED = ` ${ADDR_A} `; + const UPPER = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PEER_A}`; + const LOWER = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PEER_A}`; + + function db(): Database { + const d = new Database(':memory:'); + initLISHnetsTables(d); + return d; + } + + it('add() trims and drops blanks', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.add({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED, '', ' '], enabled: false, created: '' }); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('addIfNotExists() collapses two spellings of one address', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.addIfNotExists({ networkID: NET, name: 'A', description: '', bootstrapPeers: [UPPER, LOWER], created: '' }); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([UPPER]); + }); + + it('importNetworks() trims what it stores', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.importNetworks([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED], created: '' }]); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('importFromLISHnet() trims what it stores', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.importFromLISHnet({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED] } as any, false); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + it('replace() trims what it stores', async () => { + const d = db(); + const networks = bare(d, makeMockNet(), []); + + await networks.replace([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED], enabled: false, created: '' }]); + + expect(getLISHnet(d, NET)!.bootstrapPeers).toEqual([ADDR_A]); + }); + + /** + * Both of these used to write straight to the database with no lock at all, while + * `replaceLISHnets` deletes the whole catalog in a transaction and re-inserts the + * snapshot. An import or an add that slipped in between a queued replace's read and its + * rewrite reported success and then had its rows deleted again — and changed the ID set + * the replace had already decided its affected list from. + */ + it('an import racing a queued replace is ordered against it, not lost', async () => { + const d = db(); + const mock = makeMockNet(); + const networks = bare(d, mock, []); + const release = await (networks as any).catalogMutex.acquire(); + + const replacing = networks.replace([{ networkID: 'net-keep', name: 'K', description: '', bootstrapPeers: [], enabled: false, created: '' }]); + const importing = networks.importNetworks([{ networkID: NET, name: 'A', description: '', bootstrapPeers: [ADDR_A], created: '' }]); + const adding = networks.addIfNotExists({ networkID: 'net-add', name: 'B', description: '', bootstrapPeers: [], created: '' }); + release(); + const [, imported, added] = await Promise.all([replacing, importing, adding]); + + // Both queued behind the replace, so both survive it and both report truthfully. + expect(imported).toBe(1); + expect(added).toBe(true); + expect(getLISHnet(d, NET)).toBeDefined(); + expect(getLISHnet(d, 'net-add')).toBeDefined(); + expect(getLISHnet(d, 'net-keep')).toBeDefined(); + }); + + it('validateNetwork() reports the list in the shape it would be stored in', () => { + const networks = bare(db(), makeMockNet(), []); + + const definition = networks.validateNetwork({ networkID: NET, name: 'A', description: '', bootstrapPeers: [PADDED, ' '] } as any); + + expect(definition.bootstrapPeers).toEqual([ADDR_A]); + }); +}); diff --git a/backend/tests/unit/lishnet/leave-network.test.ts b/backend/tests/unit/lishnet/leave-network.test.ts index 9299f3982..fbc2020fa 100644 --- a/backend/tests/unit/lishnet/leave-network.test.ts +++ b/backend/tests/unit/lishnet/leave-network.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; +import { initLISHnetsTables, addLISHnet, getLISHnet } from '../../../src/db/lishnets.ts'; import { Networks } from '../../../src/lishnet/lishnets.ts'; /** @@ -20,10 +23,21 @@ interface MockNet { getTopicPeers(id: string): string[]; getRecentTopicMembers(id: string): string[]; unsubscribeTopic(id: string): void; - subscribeTopic(id: string): void; + isRunning(): boolean; + subscribeTopic(id: string): boolean; isBootstrapOrRelayPeer(pid: string): boolean; disconnectPeer(pid: string, networkID: string): Promise; pruneConfiguredBootstrapPeer(pid: string): void; + bumpBootstrapGeneration(networkID: string): void; + generationBumps: string[]; + resetBootstrapStatus(networkID: string): void; + statusResets: string[]; + pruneBootstrapAddresses(addresses: string[]): void; + prunedAddresses: string[][]; + pruneBootstrapStatus(networkID: string, keep: string[]): void; + prunedStatus: Array<{ networkID: string; keep: string[] }>; + addBootstrapPeers(peers: string[], networkID: string, origin: string): Promise; + dialledLists: Array<{ networkID: string; peers: string[]; origin: string }>; clearRedialSuppressionForNetwork(networkID: string): void; suppressionClearedFor: string[]; } @@ -38,6 +52,11 @@ function makeMockNet(): MockNet { bootstrapOrRelay: new Set(), prunedBootstrap: [], suppressionClearedFor: [], + generationBumps: [], + statusResets: [], + prunedAddresses: [], + prunedStatus: [], + dialledLists: [], getTopicPeers(id) { return this.topicPeers.get(id) ?? []; }, @@ -49,8 +68,12 @@ function makeMockNet(): MockNet { // Mirror real pubsub: after unsubscribe the topic reports no peers. this.topicPeers.delete(id); }, + isRunning() { + return true; + }, subscribeTopic(id) { this.subscribed.push(id); + return true; }, isBootstrapOrRelayPeer(pid) { return this.bootstrapOrRelay.has(pid); @@ -61,25 +84,58 @@ function makeMockNet(): MockNet { pruneConfiguredBootstrapPeer(pid) { this.prunedBootstrap.push(pid); }, + bumpBootstrapGeneration(networkID) { + this.generationBumps.push(networkID); + }, + resetBootstrapStatus(networkID) { + this.statusResets.push(networkID); + }, + pruneBootstrapAddresses(addresses) { + this.prunedAddresses.push(addresses); + }, + pruneBootstrapStatus(networkID, keep) { + this.prunedStatus.push({ networkID, keep }); + }, + async addBootstrapPeers(peers, networkID, origin) { + this.dialledLists.push({ networkID, peers, origin }); + }, clearRedialSuppressionForNetwork(networkID) { this.suppressionClearedFor.push(networkID); }, }; } -// bootstrapPeers per network id, exposed to the class via `get`. +// bootstrapPeers per network id, exposed to the class via `get`. The rows stand in for the +// database: reconcileLocked converges on what `get` says, so a transition edits the row. function makeNetworks(net: MockNet, joined: string[], configs: Record = {}): Networks { const networks = Object.create(Networks.prototype) as Networks; + const rows = new Map(); + for (const id of new Set([...joined, ...Object.keys(configs)])) rows.set(id, { networkID: id, bootstrapPeers: configs[id] ?? [], enabled: joined.includes(id) }); (networks as any).network = net; + (networks as any).rows = rows; (networks as any).joinedNetworks = new Set(joined); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + // Same seeding startEnabledNetworks does: already-joined at construction time. + (networks as any).announcedJoined = new Map(joined.map(id => [id, true])); (networks as any)._onNetworkLeft = null; (networks as any)._onNetworkJoined = null; - (networks as any).get = (id: string) => (configs[id] ? { networkID: id, bootstrapPeers: configs[id] } : undefined); + (networks as any).get = (id: string) => rows.get(id); return networks; } -const leave = (networks: Networks, id: string): Promise => (networks as any).leaveNetwork(id); -const join = (networks: Networks, id: string): Promise => (networks as any).joinNetwork(id); +// Both go through reconcileLocked(), which is where the join/leave notifications live. +// Same shape the real writers use: write the row, then converge the runtime on it. +function transition(networks: Networks, id: string, enabled: boolean): Promise { + const n = networks as any; + const row = n.rows.get(id); + const previous = row ? { ...row } : undefined; + if (row) row.enabled = enabled; + else n.rows.set(id, { networkID: id, bootstrapPeers: [], enabled }); + return n.operationLock(id).runExclusive(() => n.reconcileLocked(id, previous)); +} +const leave = (networks: Networks, id: string): Promise => transition(networks, id, false); +const join = (networks: Networks, id: string): Promise => transition(networks, id, true); describe('Networks.leaveNetwork — exclusive peer disconnect', () => { let net: MockNet; @@ -227,6 +283,54 @@ describe('Networks.leaveNetwork — exclusive peer disconnect', () => { expect(net.prunedBootstrap).toEqual(['pBootA']); // exemption still pruned }); + /** + * Cleanup keyed on the identity alone cannot express "this peer is configured in both + * networks, under two different addresses". The identity is in use elsewhere, so the + * whole cleanup is skipped and the left network's own address goes on counting as a + * configured bootstrap — dialed by the parked probe and exempt from removal. + */ + it('drops the left network address of a peer configured elsewhere under another address', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a', 'net-b'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + 'net-b': ['/ip4/192.0.2.2/tcp/9090/p2p/pShared'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([['/ip4/192.0.2.1/tcp/9090/p2p/pShared']]); + expect(net.prunedBootstrap).toEqual([]); // the identity itself is still configured + }); + + it('keeps an address the still-joined network configures identically', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a', 'net-b'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + 'net-b': ['/ip4/192.0.2.1/tcp/9090/p2p/pShared'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([[]]); + }); + + it('drops every address of a network left with nothing else configured', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a'], { + 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA', '/ip4/192.0.2.2/tcp/9090/p2p/pAlsoA'], + }); + await leave(networks, 'net-a'); + expect(net.prunedAddresses).toEqual([['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA', '/ip4/192.0.2.2/tcp/9090/p2p/pAlsoA']]); + }); + + /** + * The rows describe a membership that has ended. Kept, a later rejoin opens on the + * previous session's connected/error/discovered results until fresh dials happen to + * overwrite each one. + */ + it('drops the bootstrap status of the network it left', async () => { + net.topicPeers.set('net-a', []); + const networks = makeNetworks(net, ['net-a'], { 'net-a': ['/ip4/192.0.2.1/tcp/9090/p2p/pOnlyA'] }); + await leave(networks, 'net-a'); + expect(net.statusResets).toEqual(['net-a']); + }); + it('keeps a left-lishnet bootstrap peer that is still an active circuit relay', async () => { net.topicPeers.set('net-a', []); net.bootstrapOrRelay.add('pRelayNode'); // still relaying another connection @@ -272,3 +376,184 @@ describe('Networks.joinNetwork — onNetworkJoined notification', () => { expect(net.suppressionClearedFor).toEqual(['net-a']); }); }); + +/** + * The bootstrap list can be edited from two screens: the participants view, which + * calls updateBootstrapPeers, and the ordinary "edit network" form, which calls + * update. Only the first used to reach the running node, so a list changed through + * the form was written to the database and then ignored until restart — the node + * kept dialing the peers the user had just removed. + */ +describe('Networks.update — a changed bootstrap list reaches the running node', async () => { + const NET = 'net-a'; + const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function seeded(bootstrapPeers: string[], enabled = true) { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers, enabled, created: '2026-01-01T00:00:00.000Z' }); + const mock = makeMockNet(); + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).network = mock; + (networks as any).db = db; + (networks as any).joinedNetworks = new Set(enabled ? [NET] : []); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + (networks as any).announcedJoined = new Map(enabled ? [[NET, true]] : []); + return { networks, mock, db }; + } + + const edit = (networks: Networks, bootstrapPeers: string[], enabled = true): Promise => (networks as any).update({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled, created: '2026-01-01T00:00:00.000Z' }); + + it('prunes the status and dials the new list when the entries change', async () => { + const { networks, mock } = seeded([ADDR_A]); + await edit(networks, [ADDR_B]); + expect(mock.prunedStatus).toEqual([{ networkID: NET, keep: [ADDR_B] }]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_B], origin: 'configured' }]); + }); + + it('drops the bootstrap exemption of a peer removed through the form', async () => { + const { networks, mock } = seeded([ADDR_A, ADDR_B]); + await edit(networks, [ADDR_A]); + expect(mock.prunedBootstrap).toEqual([PEER_B]); + }); + + it('leaves the running node alone when only the name changed', async () => { + const { networks, mock } = seeded([ADDR_A]); + await edit(networks, [ADDR_A]); + expect(mock.prunedStatus).toEqual([]); + expect(mock.dialledLists).toEqual([]); + expect(mock.prunedBootstrap).toEqual([]); + }); + + /** + * The form can submit blank rows. Persisting them raw while the runtime worked from + * the filtered copy left the database and the live node disagreeing about what the + * network's bootstrap list actually is. + */ + it('persists the cleaned list, not the blank rows the form submitted', async () => { + const { networks, db } = seeded([ADDR_A]); + await edit(networks, ['', ADDR_B, ' ']); + expect(getLISHnet(db, NET)?.bootstrapPeers).toEqual([ADDR_B]); + }); + + /** + * Editing only the host or port keeps the peer ID, so the identity-level prune sees + * nothing to do. Without an address-level prune the replaced address stays on the + * autodial list and recovery keeps dialing it. + */ + it('drops the replaced address when only the host changed', async () => { + const moved = `/ip4/203.0.113.99/tcp/9090/p2p/${PEER_A}`; + const { networks, mock } = seeded([ADDR_A]); + await edit(networks, [moved]); + expect(mock.prunedAddresses).toEqual([[ADDR_A]]); + expect(mock.prunedBootstrap).toEqual([]); + }); + + it('keeps an address that is still configured for another joined network', async () => { + const { networks, mock } = seeded([ADDR_A]); + (networks as any).get = (nid: string) => (nid === 'net-other' ? { networkID: nid, bootstrapPeers: [ADDR_A] } : { networkID: NET, bootstrapPeers: [ADDR_A] }); + (networks as any).joinedNetworks = new Set([NET, 'net-other']); + await edit(networks, [ADDR_B]); + expect(mock.prunedAddresses).toEqual([[]]); + }); + + /** + * The autodial prune compares addresses canonically, so the "did this entry leave + * the list" check has to as well — otherwise one spelling of an address counts as a + * removal here and as the same entry there. + */ + it('treats two spellings of one address as the same entry', async () => { + const upper = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PEER_A}`; + const lower = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PEER_A}`; + const { networks, mock } = seeded([upper]); + await edit(networks, [lower]); + expect(mock.prunedAddresses).toEqual([[]]); + }); + + it('does not dial for a network that is not joined', async () => { + const { networks, mock } = seeded([ADDR_A], false); + await edit(networks, [ADDR_B], false); + expect(mock.dialledLists).toEqual([]); + expect(mock.prunedStatus).toHaveLength(1); + }); + + /** + * The edit form carries the enabled flag too, so it can turn a network on or off — + * and that has to reach the node, not just the database row. + */ + it('joins a network the edit enabled', async () => { + const { networks, mock } = seeded([ADDR_A], false); + await edit(networks, [ADDR_A], true); + expect(mock.subscribed).toEqual([NET]); + expect(mock.dialledLists).toEqual([{ networkID: NET, peers: [ADDR_A], origin: 'configured' }]); + }); + + it('leaves a network the edit disabled', async () => { + const { networks, mock } = seeded([ADDR_A], true); + await edit(networks, [ADDR_A], false); + expect(mock.unsubscribed).toEqual([NET]); + }); +}); + +/** + * The disable path writes the row first and reconciles the runtime afterwards, so the + * leave cannot ask the database what it is leaving — by then the row holds the INCOMING + * list, or has been deleted outright. It used to do exactly that, which meant the old + * addresses and identities kept their bootstrap exemption on the live node: still + * force-dialled, still exempt from the stale sweep, still reconnectable after removal. + */ +describe('Networks — leaving cleans the configuration it is leaving, not the new one', () => { + const NET = 'net-a'; + const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + const ROW = (bootstrapPeers: string[], enabled: boolean) => ({ networkID: NET, name: 'A', description: '', bootstrapPeers, enabled, created: '2026-01-01T00:00:00.000Z' }); + + /** A joined network with `bootstrapPeers`, over a real in-memory row. */ + function joined(bootstrapPeers: string[]) { + const db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, ROW(bootstrapPeers, true)); + const mock = makeMockNet(); + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).network = mock; + (networks as any).db = db; + (networks as any).joinedNetworks = new Set([NET]); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + (networks as any).announcedJoined = new Map([[NET, true]]); + return { networks, mock, db }; + } + + it('an edit that swaps the list and disables at once still prunes the old address', async () => { + const { networks, mock } = joined([ADDR_A]); + await (networks as any).update(ROW([ADDR_B], false)); + expect(mock.unsubscribed).toEqual([NET]); + // Re-reading the row gave [ADDR_B] here, so ADDR_A was never pruned and PEER_A + // kept its exemption while PEER_B — a peer we never joined with — lost its own. + expect(mock.prunedAddresses.flat()).toEqual([ADDR_A]); + expect(mock.prunedBootstrap).toEqual([PEER_A]); + }); + + it('a replace() that removes a joined network prunes its bootstraps', async () => { + const { networks, mock } = joined([ADDR_A]); + await networks.replace([]); + expect(mock.unsubscribed).toEqual([NET]); + // The row is gone by the time the leave runs, so a re-read yielded nothing at all. + expect(mock.prunedAddresses.flat()).toEqual([ADDR_A]); + expect(mock.prunedBootstrap).toEqual([PEER_A]); + }); + + it('a replace() that changes the list and disables at once prunes the old address', async () => { + const { networks, mock } = joined([ADDR_A]); + await networks.replace([ROW([ADDR_B], false)]); + expect(mock.unsubscribed).toEqual([NET]); + expect(mock.prunedAddresses.flat()).toEqual([ADDR_A]); + expect(mock.prunedBootstrap).toEqual([PEER_A]); + }); +}); diff --git a/backend/tests/unit/lishnet/start-enabled-networks.test.ts b/backend/tests/unit/lishnet/start-enabled-networks.test.ts new file mode 100644 index 000000000..1aa21d589 --- /dev/null +++ b/backend/tests/unit/lishnet/start-enabled-networks.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, beforeEach } from 'bun:test'; +import { Database } from 'bun:sqlite'; +import { Mutex } from 'async-mutex'; +import { initLISHnetsTables, addLISHnet, deleteLISHnet, setLISHnetEnabled } from '../../../src/db/lishnets.ts'; +import { Networks } from '../../../src/lishnet/lishnets.ts'; + +/** + * Startup used to snapshot the enabled networks BEFORE the (slow) node start and then + * subscribe from that copy. Anything the API did during the start — disable, delete, or + * a full stop — was reconciled against a runtime that had joined nothing yet, so it had + * nothing to undo, and the loop went on to join the network from its stale list. + */ + +const NET = 'net-a'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +/** A Network stub whose start() can be held open, tracking subscribe/unsubscribe. */ +function makeMockNet(startGate: Promise) { + return { + subscribed: [] as string[], + unsubscribed: [] as string[], + running: false, + async start(): Promise { + await startGate; + this.running = true; + }, + async stop(): Promise { + this.running = false; + }, + isRunning(): boolean { + return this.running; + }, + subscribeTopic(id: string): boolean { + this.subscribed.push(id); + return true; + }, + unsubscribeTopic(id: string): void { + this.unsubscribed.push(id); + }, + getTopicPeers: (): string[] => [], + getRecentTopicMembers: (): string[] => [], + isBootstrapOrRelayPeer: (): boolean => false, + disconnectPeer: async (): Promise => {}, + pruneConfiguredBootstrapPeer(): void {}, + resetBootstrapStatus(): void {}, + pruneBootstrapAddresses(): void {}, + pruneBootstrapStatus(): void {}, + clearRedialSuppressionForNetwork(): void {}, + addBootstrapPeers: async (): Promise => {}, + }; +} + +function makeNetworks(net: ReturnType, db: Database): Networks { + const networks = Object.create(Networks.prototype) as Networks; + (networks as any).db = db; + (networks as any).network = net; + (networks as any).joinedNetworks = new Set(); + (networks as any).networkOperations = new Map(); + (networks as any).catalogMutex = new Mutex(); + (networks as any).announcedJoined = new Map(); + (networks as any).shuttingDown = false; + (networks as any)._onNetworkJoined = null; + (networks as any)._onNetworkLeft = null; + return networks; +} + +/** Let the startup loop get past its awaits. */ +async function settle(): Promise { + for (let i = 0; i < 12; i++) await Promise.resolve(); +} + +describe('Networks.startEnabledNetworks — coordinated with concurrent changes', () => { + let db: Database; + + beforeEach(() => { + db = new Database(':memory:'); + initLISHnetsTables(db); + addLISHnet(db, { networkID: NET, name: 'A', description: '', bootstrapPeers: [], enabled: true, created: '2026-01-01T00:00:00.000Z' }); + }); + + it('does not join a network disabled while the node was starting', async () => { + const gate = deferred(); + const net = makeMockNet(gate.promise); + const networks = makeNetworks(net, db); + + const starting = networks.startEnabledNetworks(); + await settle(); + // The API disables the network while start() is still outstanding. + setLISHnetEnabled(db, NET, false); + gate.resolve(); + await starting; + + expect(net.subscribed).toEqual([]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('does not join a network deleted while the node was starting', async () => { + const gate = deferred(); + const net = makeMockNet(gate.promise); + const networks = makeNetworks(net, db); + + const starting = networks.startEnabledNetworks(); + await settle(); + deleteLISHnet(db, NET); + gate.resolve(); + await starting; + + expect(net.subscribed).toEqual([]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('does not claim a network as joined once a stop has begun', async () => { + const gate = deferred(); + const net = makeMockNet(gate.promise); + const networks = makeNetworks(net, db); + + const starting = networks.startEnabledNetworks(); + await settle(); + gate.resolve(); + // The stop lands the moment the node is up, before the subscribe loop runs. + const stopping = networks.stopAllNetworks(); + await Promise.all([starting, stopping]); + + // Claiming membership of a topic on a stopped node is the failure this guards. + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + expect(net.subscribed).toEqual([]); + }); + + it('a stop forgets what it had announced, so the next change is announced again', async () => { + const net = makeMockNet(Promise.resolve()); + const networks = makeNetworks(net, db); + const events: string[] = []; + (networks as any)._onNetworkJoined = (id: string): void => { + events.push(`joined:${id}`); + }; + + await networks.startEnabledNetworks(); + expect((networks as any).announcedJoined.get(NET)).toBe(true); + + await networks.stopAllNetworks(); + // Surviving the stop, the `true` here made the rejoin below look like no change at + // all — the runtime had gone down and come back and nobody was told. + expect((networks as any).announcedJoined.has(NET)).toBe(false); + + // The node has to be back up before an enable can join anything at all, and it + // comes back with the network disabled — the case that used to be silent. + setLISHnetEnabled(db, NET, false); + await networks.startEnabledNetworks(); + expect((networks as any).announcedJoined.has(NET)).toBe(false); + + await networks.setEnabled(NET, true); + expect(events).toEqual([`joined:${NET}`]); + }); + + /** + * stopAllNetworks used to set a flag and stop the node without waiting for, or blocking, + * the per-network operations. An enable queued behind a slow one then woke up after the + * stop, subscribed a dead pubsub (a logged no-op) and recorded the network as joined + * anyway — a membership with no subscription that the next startup skips as "already + * joined". + */ + it('an enable that arrives during a shutdown joins nothing', async () => { + const net = makeMockNet(Promise.resolve()); + const networks = makeNetworks(net, db); + await networks.startEnabledNetworks(); + await networks.setEnabled(NET, false); + const subscribedBefore = [...net.subscribed]; + + const stopping = networks.stopAllNetworks(); + const enabling = networks.setEnabled(NET, true); + await Promise.all([stopping, enabling]); + + expect(net.running).toBe(false); + expect(net.subscribed).toEqual(subscribedBefore); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('a stop that fails keeps the membership it could not prove gone', async () => { + const net = makeMockNet(Promise.resolve()); + const networks = makeNetworks(net, db); + await networks.startEnabledNetworks(); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + + net.stop = async (): Promise => { + throw new Error('node.stop failed'); + }; + await expect(networks.stopAllNetworks()).rejects.toThrow('node.stop failed'); + + // Discarding the membership before the node was proved down left `leaveNetwork()` + // with "not joined, nothing to do", so the disable below wrote `enabled=false` and + // then unsubscribed nothing on a node that is still alive and still in the topic. + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + await networks.setEnabled(NET, false); + expect(net.unsubscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(false); + }); + + it('an undisturbed startup still joins every enabled network', async () => { + const net = makeMockNet(Promise.resolve()); + const networks = makeNetworks(net, db); + + await networks.startEnabledNetworks(); + + expect(net.subscribed).toEqual([NET]); + expect((networks as any).joinedNetworks.has(NET)).toBe(true); + expect((networks as any).announcedJoined.get(NET)).toBe(true); + }); +}); diff --git a/backend/tests/unit/protocol/bootstrap-status.test.ts b/backend/tests/unit/protocol/bootstrap-status.test.ts index 4ad293c83..c76840d7d 100644 --- a/backend/tests/unit/protocol/bootstrap-status.test.ts +++ b/backend/tests/unit/protocol/bootstrap-status.test.ts @@ -1,5 +1,7 @@ import { describe, it, expect } from 'bun:test'; -import { classifyBootstrapError, extractActualPeerID } from '../../../src/protocol/network.ts'; +import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; +import { classifyBootstrapError, extractActualPeerID, extractDestinationPeerID } from '../../../src/protocol/network.ts'; +import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; // Deterministic unit tests for the bootstrap-peer dial classification — the pure // logic that decides whether a failed bootstrap dial is an identity-mismatch (stale @@ -57,3 +59,690 @@ describe('extractActualPeerID', () => { expect(extractActualPeerID('does not match expected remote identity key only')).toBe(null); }); }); + +describe('extractDestinationPeerID', () => { + // Real base58 ed25519 peer IDs are required — the multiaddr parser validates them. + const RELAY_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const DST_ID = '12D3KooWAnfqA6Wap96ixVfxhHeGUDMriBG4Nncp5tqu8q71EVv2'; + + it('returns the terminal peer ID of a plain address', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${DST_ID}`))).toBe(DST_ID); + }); + + it('returns the DESTINATION (not the relay) for a circuit address', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${RELAY_ID}/p2p-circuit/p2p/${DST_ID}`))).toBe(DST_ID); + }); + + it('returns the relay ID when a circuit address has no destination component', () => { + expect(extractDestinationPeerID(Multiaddr(`/ip4/192.0.2.1/tcp/9090/p2p/${RELAY_ID}/p2p-circuit`))).toBe(RELAY_ID); + }); + + it('returns null for an address without any peer ID and for garbage input', () => { + expect(extractDestinationPeerID(Multiaddr('/ip4/192.0.2.1/tcp/9090'))).toBe(null); + expect(extractDestinationPeerID(null)).toBe(null); + }); +}); + +describe('BootstrapStatusTracker.deleteDiscoveredByPeerID', () => { + const NET_A = 'netAAAA'; + const NET_B = 'netBBBB'; + const DEAD_ID = '12D3KooWDeadDeadDeadDeadDeadDeadDeadDeadDeadDeadDD'; + const LIVE_ID = '12D3KooWLiveLiveLiveLiveLiveLiveLiveLiveLiveLiveLL'; + const DEAD_ADDR_1 = `/ip4/192.0.2.10/tcp/9090/p2p/${DEAD_ID}`; + const DEAD_ADDR_2 = `/ip4/192.0.2.11/tcp/9090/p2p/${DEAD_ID}`; + const LIVE_ADDR = `/ip4/192.0.2.20/tcp/9090/p2p/${LIVE_ID}`; + + it('removes discovered rows for the peer across all networks, keeps other peers', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, DEAD_ADDR_1, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + tracker.recordOutcome(NET_A, LIVE_ADDR, LIVE_ID, 'connected', null, null, 'discovered'); + tracker.recordOutcome(NET_B, DEAD_ADDR_2, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)?.peers.map(p => p.multiaddr)).toEqual([LIVE_ADDR]); + expect(tracker.getStatus(NET_B)).toBe(null); // network map emptied entirely + }); + + it('keeps configured rows for the same peer identity', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, DEAD_ADDR_1, DEAD_ID, 'timeout', 'The operation timed out', null, 'configured'); + tracker.recordOutcome(NET_A, DEAD_ADDR_2, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)?.peers.map(p => p.multiaddr)).toEqual([DEAD_ADDR_1]); + }); + + it('matches rows by actualPeerID as well and fires onStatusChange per changed network', () => { + const tracker = new BootstrapStatusTracker(); + const events: string[] = []; + tracker.setOnChange(networkID => events.push(networkID)); + // Row whose expectedPeerID is null but whose dial revealed the dead identity. + tracker.recordOutcome(NET_A, '/ip4/192.0.2.30/tcp/9090', null, 'identity-mismatch', 'mismatch', DEAD_ID, 'discovered'); + tracker.recordOutcome(NET_B, LIVE_ADDR, LIVE_ID, 'connected', null, null, 'discovered'); + events.length = 0; + + tracker.deleteDiscoveredByPeerID(DEAD_ID); + + expect(tracker.getStatus(NET_A)).toBe(null); + expect(events).toEqual([NET_A]); // untouched NET_B emits nothing + }); +}); + +describe('BootstrapStatusTracker.sweepStale', () => { + const NET = 'netAAAA'; + const TTL = 30 * 60_000; + const DEAD_ID = '12D3KooWDeadDeadDeadDeadDeadDeadDeadDeadDeadDeadDD'; + const LIVE_ID = '12D3KooWLiveLiveLiveLiveLiveLiveLiveLiveLiveLiveLL'; + const DEAD_ADDR = `/ip4/192.0.2.10/tcp/9090/p2p/${DEAD_ID}`; + const LIVE_ADDR = `/ip4/192.0.2.20/tcp/9090/p2p/${LIVE_ID}`; + const CONF_ADDR = `/ip4/192.0.2.30/tcp/9090/p2p/${DEAD_ID}`; + + it('drops stale discovered rows, keeps fresh, connected and configured ones', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + // The live row carries the identity its dial actually proved, which is what the + // production path records on a successful connection. + tracker.recordOutcome(NET, LIVE_ADDR, LIVE_ID, 'connected', null, LIVE_ID, 'discovered'); + tracker.recordOutcome(NET, CONF_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'configured'); + const past = Date.now() + TTL + 60_000; // both rows are then older than TTL + + tracker.sweepStale(TTL, (_net, pid) => pid === LIVE_ID, past); + + const addrs = tracker + .getStatus(NET) + ?.peers.map(p => p.multiaddr) + .sort(); + // DEAD discovered row expired; LIVE row survives via membership; configured row untouchable. + expect(addrs).toEqual([CONF_ADDR, LIVE_ADDR].sort()); + }); + + it('drops a row frozen at connected once the peer is no longer a network member', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, () => false, Date.now() + TTL + 60_000); + + expect(tracker.getStatus(NET)).toBe(null); + }); + + it('expires a row whose peer stays globally connected but left THIS network', () => { + // Membership predicate returns false for NET even though the peer is up + // elsewhere — the stale NET row must still expire past its TTL. + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, networkID => networkID !== NET, Date.now() + TTL + 60_000); + + expect(tracker.getStatus(NET)).toBe(null); + }); + + /** + * A discovered multiaddr practically always carries a /p2p/, so honouring the + * CLAIMED identity here let anyone keep a row alive forever by naming a live member + * in an address nothing ever answered on. + */ + it('expires an address that merely claims a live member without ever answering', () => { + const tracker = new BootstrapStatusTracker(); + const invented = `/ip4/198.51.100.77/tcp/9090/p2p/${LIVE_ID}`; + tracker.recordOutcome(NET, invented, LIVE_ID, 'timeout', 'no answer', null, 'discovered'); + + tracker.sweepStale(TTL, (_net, pid) => pid === LIVE_ID, Date.now() + TTL + 60_000); + + expect(tracker.getStatus(NET)).toBe(null); + }); + + it('keeps rows within the TTL even for a non-member', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + tracker.sweepStale(TTL, () => false); // real clock — row was written moments ago + + expect(tracker.getStatus(NET)?.peers.length).toBe(1); + }); + + /** + * markPending fires whenever gossip names an address again, which happens on every + * announce cycle — far more often than the sweep TTL. Treating that as activity kept + * a dead peer's row alive forever. Only a dial that produced an outcome counts. + */ + // Rows are stamped with `new Date()`, which no Date.now stub can steer, so these + // read the real timestamp the tracker wrote and drive sweepStale relative to it. + // The short sleep only guarantees a measurable gap between the two writes. + const clockOf = (tracker: BootstrapStatusTracker): string => tracker.getStatus(NET)!.peers[0]!.updatedAt; + + it('does not let a re-mention refresh the staleness clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const outcomeAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.markPending(NET, DEAD_ADDR, DEAD_ID, 'discovered'); // gossip mentions it again + + expect(clockOf(tracker)).toBe(outcomeAt); // clock untouched by the mention + tracker.sweepStale(TTL, () => false, Date.parse(outcomeAt) + TTL + 2); + expect(tracker.getStatus(NET)).toBe(null); // ages out from the last real outcome + }); + + /** + * The failure is this node's own reaction to somebody else's mention of a dead peer. + * Counting it as activity was the same immortality bug as the mention itself: gossip + * names the peer, the dial fails, the row is refreshed, and the TTL never arrives. + */ + it('does not let a failed dial outcome refresh the staleness clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const firstAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + + expect(Date.parse(clockOf(tracker))).toBeGreaterThan(Date.parse(firstAt)); // display clock moves + tracker.sweepStale(TTL, () => false, Date.parse(firstAt) + TTL + 2); + expect(tracker.getStatus(NET)).toBe(null); // staleness clock did not + }); + + it('lets a successful dial refresh the staleness clock', async () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'timeout', 'The operation timed out', null, 'discovered'); + const firstAt = clockOf(tracker); + await Bun.sleep(5); + + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + tracker.sweepStale(TTL, () => false, Date.parse(firstAt) + TTL + 2); + expect(tracker.getStatus(NET)?.peers.length).toBe(1); // survives — the address answered + }); + + /** The clock is bookkeeping, not part of what the API hands out. */ + it('keeps the staleness clock out of the published snapshot', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, DEAD_ADDR, DEAD_ID, 'connected', null, null, 'discovered'); + + expect(tracker.getStatus(NET)!.peers[0]!).not.toHaveProperty('staleSince'); + }); + + it('starts the clock on a first mention that has no prior row', () => { + const tracker = new BootstrapStatusTracker(); + tracker.markPending(NET, DEAD_ADDR, DEAD_ID, 'discovered'); + const mentionAt = clockOf(tracker); + + tracker.sweepStale(TTL, () => false, Date.parse(mentionAt) + TTL - 60_000); + expect(tracker.getStatus(NET)?.peers.length).toBe(1); // inside the TTL + + tracker.sweepStale(TTL, () => false, Date.parse(mentionAt) + TTL + 2); + expect(tracker.getStatus(NET)).toBe(null); // and expires once past it + }); +}); + +/** + * Every mutation rebuilds and emits the whole peer list, and intake of one announce + * performs two per address. batch() groups them so the UI receives one snapshot for + * the run instead of one per row per address. + */ +describe('BootstrapStatusTracker.batch', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWBatchBatchBatchBatchBatchBatchBatchBatchBB'; + const addr = (i: number): string => `/ip4/192.0.2.${i}/tcp/9090/p2p/${PID}`; + + function tracked() { + const tracker = new BootstrapStatusTracker(); + const seen: string[][] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.map(p => p.multiaddr))); + return { tracker, seen }; + } + + it('emits exactly one snapshot for many mutations', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => { + for (let i = 1; i <= 10; i++) { + tracker.markPending(NET, addr(i), PID, 'discovered'); + tracker.recordOutcome(NET, addr(i), PID, 'connected', null, null, 'discovered'); + } + }); + + expect(seen.length).toBe(1); + expect(seen[0]!.length).toBe(10); // the one snapshot holds every row + }); + + it('still emits when the body throws', () => { + const { tracker, seen } = tracked(); + + expect(() => + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial loop blew up'); + }) + ).toThrow('dial loop blew up'); + + expect(seen).toEqual([[addr(1)]]); + }); + + it('holds the frame open across awaits and emits once the promise settles', async () => { + const { tracker, seen } = tracked(); + + const done = tracker.batch(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + await Promise.resolve(); + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + expect(seen).toEqual([]); // nothing emitted while the body is still running + await done; + + expect(seen).toEqual([[addr(1), addr(2)]]); + }); + + it('emits once when a rejected async body settles', async () => { + const { tracker, seen } = tracked(); + + const done = tracker.batch(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial rejected'); + }); + + await expect(done).rejects.toThrow('dial rejected'); + expect(seen).toEqual([[addr(1)]]); + }); + + it('emits nothing when the body changed nothing', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => {}); + + expect(seen).toEqual([]); + }); + + it('collapses nested batches of the same network into one snapshot', () => { + const { tracker, seen } = tracked(); + + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + tracker.batch(NET, () => { + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + expect(seen).toEqual([]); // inner exit must not publish a half-built run + }); + + expect(seen).toEqual([[addr(1), addr(2)]]); + }); + + it('leaves single-mutation callers emitting per mutation', () => { + const { tracker, seen } = tracked(); + + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + + expect(seen.length).toBe(2); + }); + + it('does not defer mutations of a different network', () => { + const { tracker, seen } = tracked(); + const OTHER = 'netBBBB'; + + tracker.batch(NET, () => { + tracker.recordOutcome(OTHER, addr(1), PID, 'connected', null, null, 'discovered'); + expect(seen.length).toBe(1); // the other network is not part of this batch + }); + + expect(seen.length).toBe(1); // NET itself changed nothing → no second emit + }); + + it('returns the body value unchanged', () => { + const { tracker } = tracked(); + expect(tracker.batch(NET, () => 42)).toBe(42); + }); +}); + +describe('BootstrapStatusTracker discovered-row cap', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWCapCapCapCapCapCapCapCapCapCapCapCapCapCapCA'; + + it('bounds discovered rows per network and keeps configured rows', () => { + const tracker = new BootstrapStatusTracker(); + // One configured row that must always survive. + tracker.recordOutcome(NET, `/ip4/198.51.100.1/tcp/9090/p2p/${PID}`, PID, 'connected', null, null, 'configured'); + // Flood well past the 256 cap with unique discovered addresses. + for (let i = 0; i < 400; i++) tracker.recordOutcome(NET, `/ip4/203.0.113.${i % 254}/tcp/${9000 + i}/p2p/${PID}`, PID, 'connected', null, null, 'discovered'); + + const peers = tracker.getStatus(NET)!.peers; + const discovered = peers.filter(p => p.origin === 'discovered').length; + const configured = peers.filter(p => p.origin === 'configured').length; + expect(discovered).toBeLessThanOrEqual(256); + expect(configured).toBe(1); + }); +}); + +/** + * pruneEntries is fed the network's CONFIGURED bootstrap list after the user edits it. + * Since this tracker also holds gossip-discovered rows, judging every row by that list + * would empty the participant view on a bootstrap edit — including a "refresh from + * public list" — leaving it blank until gossip mentions each peer again. + */ +describe('BootstrapStatusTracker.pruneEntries', () => { + const NET = 'netAAAA'; + const CONF_KEPT = '/ip4/192.0.2.1/tcp/9090/p2p/12D3KooWConfKeptKeptKeptKeptKeptKeptKeptKeptKeptK'; + const CONF_DROPPED = '/ip4/192.0.2.2/tcp/9090/p2p/12D3KooWConfGoneGoneGoneGoneGoneGoneGoneGoneGone'; + const DISCOVERED = '/ip4/192.0.2.3/tcp/9090/p2p/12D3KooWDiscDiscDiscDiscDiscDiscDiscDiscDiscDis'; + + function seeded(): BootstrapStatusTracker { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, CONF_KEPT, null, 'connected', null, null, 'configured'); + tracker.recordOutcome(NET, CONF_DROPPED, null, 'connected', null, null, 'configured'); + tracker.recordOutcome(NET, DISCOVERED, null, 'connected', null, null, 'discovered'); + return tracker; + } + + const addresses = (tracker: BootstrapStatusTracker): string[] => (tracker.getStatus(NET)?.peers ?? []).map(p => p.multiaddr); + + it('drops a configured row that left the config', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).not.toContain(CONF_DROPPED); + }); + + it('keeps a configured row that is still in the config', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).toContain(CONF_KEPT); + }); + + it('keeps discovered rows, which the configured list never mentions', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(addresses(tracker)).toContain(DISCOVERED); + }); + + it('keeps discovered rows even when the whole config is cleared', () => { + const tracker = seeded(); + tracker.pruneEntries(NET, []); + expect(addresses(tracker)).toEqual([DISCOVERED]); + }); + + /** + * Removing the last row drops the whole network from the tracker, at which point + * buildStatus has nothing to return. Staying silent there would leave the UI + * rendering the very row that was just deleted, so the empty list is emitted + * explicitly — the same fallback the other removal paths use. + */ + it('emits an empty list when the last remaining row is removed', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, CONF_KEPT, null, 'connected', null, null, 'configured'); + const seen: Array<{ networkID: string; peers: unknown[] }> = []; + tracker.setOnChange((networkID, status) => seen.push({ networkID, peers: status.peers })); + tracker.pruneEntries(NET, []); + expect(seen).toEqual([{ networkID: NET, peers: [] }]); + }); + + it('still emits the surviving rows when only some are removed', () => { + const tracker = seeded(); + const seen: string[][] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.map(p => p.multiaddr))); + tracker.pruneEntries(NET, [CONF_KEPT]); + expect(seen).toEqual([[CONF_KEPT, DISCOVERED]]); + }); +}); + +/** + * Bootstrap intake awaits a dial between each address's pending mark and its outcome, so + * neither of the simple answers works: emitting per mutation costs a whole snapshot per + * row per address, and holding everything to the end of the list leaves the UI blank for + * as long as the dials take. + */ +describe('BootstrapStatusTracker.batchDebounced', () => { + const NET = 'netAAAA'; + const PID = '12D3KooWBatchBatchBatchBatchBatchBatchBatchBatchBB'; + const addr = (i: number): string => `/ip4/192.0.2.${i}/tcp/9090/p2p/${PID}`; + + function tracked() { + const tracker = new BootstrapStatusTracker(); + const seen: number[] = []; + tracker.setOnChange((_networkID, status) => seen.push(status.peers.length)); + return { tracker, seen }; + } + + it('collapses a fast run of mutations into a single emission', async () => { + const { tracker, seen } = tracked(); + + await tracker.batchDebounced(NET, async () => { + for (let i = 0; i < 20; i++) { + tracker.markPending(NET, addr(i), PID, 'discovered'); + tracker.recordOutcome(NET, addr(i), PID, 'connected', null, null, 'discovered'); + } + }); + + expect(seen).toEqual([20]); + }); + + it('publishes progress while a slow run is still going', async () => { + const { tracker, seen } = tracked(); + + await tracker.batchDebounced(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + await Bun.sleep(200); // a dial's worth of waiting + tracker.recordOutcome(NET, addr(2), PID, 'connected', null, null, 'discovered'); + }); + + expect(seen.length).toBeGreaterThan(1); // not held back to the end + expect(seen[seen.length - 1]).toBe(2); + }); + + it('emits nothing for a run that changed nothing', async () => { + const { tracker, seen } = tracked(); + await tracker.batchDebounced(NET, async () => {}); + expect(seen).toEqual([]); + }); + + it('propagates the body result and still closes on a throw', async () => { + const { tracker, seen } = tracked(); + + await expect( + tracker.batchDebounced(NET, async () => { + tracker.recordOutcome(NET, addr(1), PID, 'connected', null, null, 'discovered'); + throw new Error('dial exploded'); + }) + ).rejects.toThrow('dial exploded'); + + expect(seen).toEqual([1]); // what it managed to change was still published + }); +}); + +/** + * The address-level probes know an endpoint answered but not which networks were waiting + * to hear it. The parked-bootstrap probe is the case that matters: it is the only thing + * that retries an address the routability filter rejected at configure time, so the red + * row it left behind had no other way back to green. + */ +describe('BootstrapStatusTracker.recordAddressReachable', () => { + const NET_A = 'netAAAA'; + const NET_B = 'netBBBB'; + const PID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const ADDR = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PID}`; + const OTHER = `/ip4/192.0.2.50/tcp/9090/p2p/${PID}`; + + function seeded() { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET_A, ADDR, PID, 'error', 'address is not routable from this host', null, 'configured'); + tracker.recordOutcome(NET_B, ADDR, PID, 'error', 'address is not routable from this host', null, 'configured'); + tracker.recordOutcome(NET_A, OTHER, PID, 'timeout', 'The operation timed out', null, 'configured'); + return tracker; + } + + const statusOf = (tracker: BootstrapStatusTracker, networkID: string, addr: string): string | undefined => tracker.getStatus(networkID)?.peers.find(p => p.multiaddr === addr)?.status; + + it('repairs the row in every network that configured the address', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(statusOf(tracker, NET_A, ADDR)).toBe('connected'); + expect(statusOf(tracker, NET_B, ADDR)).toBe('connected'); + }); + + it('clears the error text it is replacing', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(tracker.getStatus(NET_A)?.peers.find(p => p.multiaddr === ADDR)?.lastError).toBe(null); + }); + + it('leaves other addresses of the same peer alone', () => { + const tracker = seeded(); + tracker.recordAddressReachable(ADDR); + expect(statusOf(tracker, NET_A, OTHER)).toBe('timeout'); + }); + + /** The probe walks parsed multiaddrs; the rows keep the spelling the user typed. */ + it('matches the row canonically, not by string identity', () => { + const tracker = seeded(); + tracker.recordAddressReachable(`/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PID}`); + expect(statusOf(tracker, NET_A, ADDR)).toBe('connected'); + }); + + it('emits nothing when no row is waiting for that address', () => { + const tracker = seeded(); + const seen: string[] = []; + tracker.setOnChange(networkID => seen.push(networkID)); + tracker.recordAddressReachable(`/ip4/198.51.100.77/tcp/9090/p2p/${PID}`); + expect(seen).toEqual([]); + }); +}); + +/** + * Rows are keyed by the canonical form of the endpoint. Keying by the raw string let two + * spellings of one address — DNS case, a trailing dot, an expanded IPv6 literal — open + * two contradictory rows, spend the discovered budget twice, and survive a delete aimed + * at only one of them. + */ +describe('BootstrapStatusTracker — one row per endpoint, whatever the spelling', () => { + const PID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const UPPER = `/dns4/BOOTSTRAP.EXAMPLE.ORG./tcp/9090/p2p/${PID}`; + const LOWER = `/dns4/bootstrap.example.org/tcp/9090/p2p/${PID}`; + const NET = 'net-a'; + + it('folds two spellings into a single row', () => { + const tracker = new BootstrapStatusTracker(); + + tracker.recordOutcome(NET, UPPER, PID, 'error', 'boom', null, 'discovered'); + tracker.recordOutcome(NET, LOWER, PID, 'connected', null, null, 'discovered'); + + const peers = tracker.getStatus(NET)!.peers; + expect(peers).toHaveLength(1); + expect(peers[0]!.status).toBe('connected'); + }); + + it('keeps the first spelling for display, and lets a configured one replace it', () => { + const tracker = new BootstrapStatusTracker(); + + tracker.recordOutcome(NET, LOWER, PID, 'error', 'boom', null, 'discovered'); + expect(tracker.getStatus(NET)!.peers[0]!.multiaddr).toBe(LOWER); + + tracker.markPending(NET, UPPER, PID, 'configured'); + expect(tracker.getStatus(NET)!.peers[0]!.multiaddr).toBe(UPPER); + }); + + it('deletes the row whichever spelling names it', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'error', 'boom', null, 'discovered'); + + tracker.deletePeer(NET, UPPER); + + expect(tracker.getStatus(NET)).toBeNull(); + }); + + it('keeps a configured row that was re-typed in another spelling', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'connected', null, null, 'configured'); + + tracker.pruneEntries(NET, [UPPER]); + + expect(tracker.getStatus(NET)!.peers).toHaveLength(1); + }); + + it('marks the row reachable when the probe names another spelling', () => { + const tracker = new BootstrapStatusTracker(); + tracker.recordOutcome(NET, LOWER, PID, 'timeout', 'no answer', null, 'configured'); + + tracker.recordAddressReachable(UPPER); + + expect(tracker.getStatus(NET)!.peers[0]!.status).toBe('connected'); + }); +}); + +/** + * The discovered-row cap decides what survives a flood. Dropping simply the oldest row + * let an attacker (or a broken emitter) push a live, connected participant out of the + * list with a burst of freshly invented dead addresses — undoing the protection the + * stale sweep gives an active member. + */ +describe('BootstrapStatusTracker — the cap evicts the least useful row', () => { + const NET = 'net-a'; + const MEMBER = 'PeerMemberAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; + const GHOST = 'PeerGhostBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB'; + + /** Fill the network to exactly the cap with fresh, failed, non-member rows. */ + function floodToCap(tracker: BootstrapStatusTracker, count: number): void { + for (let i = 0; i < count; i++) tracker.recordOutcome(NET, `/ip4/198.51.100.${i % 254}/tcp/${9000 + i}/p2p/${GHOST}`, GHOST, 'timeout', 'no answer', null, 'discovered'); + } + + function survivors(tracker: BootstrapStatusTracker): string[] { + return (tracker.getStatus(NET)?.peers ?? []).map(p => p.multiaddr); + } + + it('drops a fresh dead address before an older connected one', () => { + const tracker = new BootstrapStatusTracker(); + const live = `/ip4/203.0.113.7/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, live, MEMBER, 'connected', null, null, 'discovered'); + // 256 further rows put the network one over the cap. + floodToCap(tracker, 256); + + expect(survivors(tracker)).toContain(live); + }); + + it('keeps an active member whose identity we have actually verified', () => { + const tracker = new BootstrapStatusTracker(); + tracker.setMembersProvider(() => new Set([MEMBER])); + const verified = `/ip4/203.0.113.8/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, verified, MEMBER, 'connected', null, MEMBER, 'discovered'); + floodToCap(tracker, 256); + + expect(survivors(tracker)).toContain(verified); + }); + + it('a flood of invented addresses claiming a member cannot evict the member', () => { + const tracker = new BootstrapStatusTracker(); + tracker.setMembersProvider(() => new Set([MEMBER])); + const verified = `/ip4/203.0.113.8/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, verified, MEMBER, 'connected', null, MEMBER, 'discovered'); + // Every one of these DECLARES the member's peer ID and none has ever answered. + // Ranking on the declared identity gave them all top rank and evicted the genuine + // row above, purely for being the oldest of the group. + for (let i = 0; i < 300; i++) tracker.recordOutcome(NET, `/ip4/198.51.100.${i % 254}/tcp/${20000 + i}/p2p/${MEMBER}`, MEMBER, 'timeout', 'no answer', null, 'discovered'); + + expect(survivors(tracker)).toContain(verified); + expect(survivors(tracker)).toHaveLength(256); + }); + + /** + * markPending fires every time gossip names an address — far more often than anything + * dials it. Clearing the verified identity there demoted a proven member's row to an + * ordinary pending one within an announce cycle, and a flood of equally-pending + * invented addresses then evicted it for being the oldest of them. + */ + it('a gossip re-mention does not cost a verified member its protection', () => { + const tracker = new BootstrapStatusTracker(); + tracker.setMembersProvider(() => new Set([MEMBER])); + const verified = `/ip4/203.0.113.8/tcp/9090/p2p/${MEMBER}`; + tracker.recordOutcome(NET, verified, MEMBER, 'connected', null, MEMBER, 'discovered'); + tracker.markPending(NET, verified, MEMBER, 'discovered'); + for (let i = 0; i < 300; i++) tracker.markPending(NET, `/ip4/198.51.100.${i % 254}/tcp/${30000 + i}/p2p/${MEMBER}`, MEMBER, 'discovered'); + + expect(survivors(tracker)).toContain(verified); + expect(survivors(tracker)).toHaveLength(256); + }); + + it('still enforces the cap', () => { + const tracker = new BootstrapStatusTracker(); + floodToCap(tracker, 300); + + expect(survivors(tracker)).toHaveLength(256); + }); +}); diff --git a/backend/tests/unit/protocol/multiaddr-utils.test.ts b/backend/tests/unit/protocol/multiaddr-utils.test.ts new file mode 100644 index 000000000..74c5da8c0 --- /dev/null +++ b/backend/tests/unit/protocol/multiaddr-utils.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from 'bun:test'; +import { canonicalMultiaddr, destinationPeerIDOf, extractDestinationPeerID } from '../../../src/protocol/multiaddr-utils.ts'; +import { multiaddr } from '@multiformats/multiaddr'; + +const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const RELAY = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + +/** + * These helpers decide whether two addresses are "the same one", which downstream + * decides whether an entry is replaced, kept, or dialed again. Getting either half + * wrong is invisible until an address quietly lingers in the autodial list. + */ +describe('canonicalMultiaddr', () => { + /** + * The regex-only version this replaced could not do this: the two spellings are one + * address, and treating them as two left a replaced bootstrap behind after an edit. + */ + it('folds an expanded IPv6 literal to its compressed form', () => { + const expanded = `/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090/p2p/${PEER_A}`; + const compressed = `/ip6/2001:db8::1/tcp/9090/p2p/${PEER_A}`; + expect(canonicalMultiaddr(expanded)).toBe(canonicalMultiaddr(compressed)); + }); + + it('folds DNS host case and the FQDN root dot', () => { + expect(canonicalMultiaddr('/dns4/EXAMPLE.COM./tcp/443')).toBe('/dns4/example.com/tcp/443'); + }); + + it('leaves a peer id alone — base58 is case-significant', () => { + expect(canonicalMultiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`)).toContain(PEER_A); + }); + + it('trims surrounding whitespace, which a text field can easily carry in', () => { + expect(canonicalMultiaddr(` /ip4/203.0.113.9/tcp/9090/p2p/${PEER_A} `)).toBe(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`); + }); + + it('keeps genuinely different addresses different', () => { + expect(canonicalMultiaddr('/ip4/203.0.113.9/tcp/80')).not.toBe(canonicalMultiaddr('/ip4/203.0.113.9/tcp/8080')); + }); + + it('returns an unparseable value trimmed rather than throwing', () => { + expect(canonicalMultiaddr(' not-a-multiaddr ')).toBe('not-a-multiaddr'); + }); + + it('is stable when applied twice', () => { + const once = canonicalMultiaddr(`/ip6/2001:0db8::0001/tcp/9090/p2p/${PEER_A}`); + expect(canonicalMultiaddr(once)).toBe(once); + }); +}); + +describe('extractDestinationPeerID', () => { + it('returns the peer id of a plain address', () => { + expect(extractDestinationPeerID(multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_A}`))).toBe(PEER_A); + }); + + /** + * The whole reason this helper exists: the first /p2p component of a circuit address + * is the relay, and protecting or tagging the relay in place of the peer we meant is + * how the wrong identity ends up in the bootstrap sets. + */ + it('returns the destination of a circuit address, not the relay', () => { + expect(extractDestinationPeerID(multiaddr(`/ip4/198.51.100.1/tcp/4001/p2p/${RELAY}/p2p-circuit/p2p/${PEER_A}`))).toBe(PEER_A); + }); + + it('returns null when the address carries no peer id', () => { + expect(extractDestinationPeerID(multiaddr('/ip4/203.0.113.9/tcp/9090'))).toBeNull(); + }); + + it('returns null instead of throwing on something that is not a multiaddr', () => { + expect(extractDestinationPeerID({} as unknown)).toBeNull(); + }); +}); + +describe('destinationPeerIDOf', () => { + it('accepts the address as a string, whitespace and all', () => { + expect(destinationPeerIDOf(` /ip4/203.0.113.9/tcp/9090/p2p/${PEER_A} `)).toBe(PEER_A); + }); + + it('returns null for an unparseable string', () => { + expect(destinationPeerIDOf('nonsense')).toBeNull(); + }); +}); diff --git a/backend/tests/unit/protocol/network-config-self-filter.test.ts b/backend/tests/unit/protocol/network-config-self-filter.test.ts new file mode 100644 index 000000000..6983ad2ea --- /dev/null +++ b/backend/tests/unit/protocol/network-config-self-filter.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from 'bun:test'; +import { generateKeyPair } from '@libp2p/crypto/keys'; +import { buildLibp2pConfig } from '../../../src/protocol/network-config.ts'; + +/** + * A bootstrap entry is "ours" only when the address DESTINATION is our own identity. + * A relayed entry `/…/p2p//p2p-circuit/p2p/` mentions us as the relay hop + * while targeting somebody else, and a substring test dropped it as self — silently + * removing a configured peer from the bootstrap list and from gossipsub's direct set. + */ + +const REMOTE = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + +async function build(bootstrapPeers: string[], myPeerID: string) { + const privateKey = await generateKeyPair('Ed25519'); + return buildLibp2pConfig({ + privateKey, + datastore: {}, + allSettings: { network: { mdnsEnabled: false } } as any, + bootstrapPeers, + myPeerID, + }); +} + +describe('buildLibp2pConfig — the self filter reads the destination', () => { + it('keeps a relayed entry that merely passes through our own identity', async () => { + const me = '12D3KooWMyOwnIdentityAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'; + const viaUs = `/ip4/198.51.100.4/tcp/9090/p2p/${me}/p2p-circuit/p2p/${REMOTE}`; + + const { bootstrapPeerIDs } = await build([viaUs], me); + + expect([...bootstrapPeerIDs]).toEqual([REMOTE]); + }); + + it('still drops an entry that really targets us', async () => { + const me = REMOTE; + const ours = `/ip4/198.51.100.4/tcp/9090/p2p/${me}`; + + const { bootstrapPeerIDs, bootstrapMultiaddrs } = await build([ours], me); + + expect([...bootstrapPeerIDs]).toEqual([]); + expect(bootstrapMultiaddrs).toEqual([]); + }); +}); diff --git a/backend/tests/unit/protocol/network-disconnect.test.ts b/backend/tests/unit/protocol/network-disconnect.test.ts index a5bf81a55..a3625173f 100644 --- a/backend/tests/unit/protocol/network-disconnect.test.ts +++ b/backend/tests/unit/protocol/network-disconnect.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'bun:test'; import { KEEP_ALIVE } from '@libp2p/interface'; import { multiaddr } from '@multiformats/multiaddr'; -import { Network } from '../../../src/protocol/network.ts'; +import { Network, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; /** * Unit tests for Network.disconnectPeer tag hygiene: hanging up a peer must @@ -19,7 +19,11 @@ function makeNetwork() { const deleted: string[] = []; const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map>(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); (network as any).node = { getConnections: () => [], peerStore: { @@ -121,6 +125,11 @@ describe('Network.runRedialMaintenance — leave-peer suppression', () => { const network = Object.create(Network.prototype) as Network; (network as any).redialBackoff = new Map(); (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).noReachableSince = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); // A reconnected peer's suppression is lifted only if it currently shares a joined // topic — model that via a pubsub whose subscribers list the "back on topic" peers. (network as any).pubsub = { @@ -174,10 +183,23 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { const dialed: string[] = []; const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-x', new Set(suppressed)]]); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); (network as any).bootstrapMultiaddrs = bootstrapMaStrs.map(s => multiaddr(s)); (network as any).recentDisconnects = []; - (network as any).bootstrapTracker = { entries: () => [] }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; (network as any).node = { + // Recovery reads connectivity itself rather than trusting the tick's snapshot. + getPeers: () => [], async dial(ma: { toString(): string }): Promise { dialed.push(ma.toString()); }, @@ -185,19 +207,51 @@ describe('Network.runZeroConnectionRecovery — leave-peer suppression', () => { return { network, dialed }; } - const run = (network: Network, connected: any[]): Promise => (network as any).runZeroConnectionRecovery(connected); + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(); it('does not dial a bootstrap peer suppressed by leave-network', async () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([PEER_ID], [ma]); - await run(network, []); + await run(network); + expect(dialed).toEqual([]); + }); + + /** + * Recovery shares the pacing records with re-dial maintenance. Without that an + * isolated node re-dialed a dead discovered peer every tick forever: maintenance + * stops counting its failures once there is no other connection to prove we are + * online, so nothing else was slowing it down. + */ + it('skips a discovered bootstrap peer inside its backoff window', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); + await run(network); + expect(dialed).toEqual([]); + }); + + it('still dials a CONFIGURED peer inside a backoff window — it is the way back in', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000 }]]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(multiaddr(ma).toString())]); + await run(network); + expect(dialed).toEqual([multiaddr(ma).toString()]); + }); + + it('skips a discovered bootstrap peer still inside its unreachable quarantine', async () => { + const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; + const { network, dialed } = bareNetwork([], [ma]); + (network as any).unreachableQuarantine = new Map([[PEER_ID, Date.now() - 60_000]]); + await run(network); expect(dialed).toEqual([]); }); it('still dials a non-suppressed bootstrap peer', async () => { const ma = `/ip4/192.0.2.1/tcp/9090/p2p/${PEER_ID}`; const { network, dialed } = bareNetwork([], [ma]); - await run(network, []); + await run(network); expect(dialed).toEqual([multiaddr(ma).toString()]); }); }); @@ -212,9 +266,20 @@ describe('Network.addBootstrapPeers — rejoin clears suppression', () => { const network = Object.create(Network.prototype) as Network; (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); (network as any).bootstrapPeerIDs = new Set(); (network as any).bootstrapMultiaddrs = []; - (network as any).bootstrapTracker = { markPending() {}, recordOutcome() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; (network as any).node = { peerId: { toString: () => 'selfID' }, getConnections: () => [], diff --git a/backend/tests/unit/protocol/network-lifecycle.test.ts b/backend/tests/unit/protocol/network-lifecycle.test.ts new file mode 100644 index 000000000..48bd0d754 --- /dev/null +++ b/backend/tests/unit/protocol/network-lifecycle.test.ts @@ -0,0 +1,365 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * Unit tests for the transactional start()/stop() lifecycle. + * + * The node used to be considered "running" the moment `this.node` was assigned — + * half-way through start(), and for the whole of stop(). That made four things + * possible, each covered below: a phantom running state after a failed start, a + * leaked datastore handle, two concurrent starts both building a node, and a stop + * that reported success while node.stop() had thrown. + * + * The last one is what `failed` exists for: a node that would not stop keeps its + * reference and its datastore, refuses a new start and every destructive operation, + * and can only be left by a stop that actually succeeds. + */ + +/** A Network with no libp2p behind it — start()/stop() orchestration only. */ +function bareNetwork(): Network { + return new Network('/nonexistent/data-dir', {} as any, { list: (): any => ({}) } as any); +} + +/** + * A stand-in for libp2p that follows its real stop state machine (libp2p 3.3.3, + * `libp2p.js`): `stop()` returns immediately unless the status is 'started', sets + * 'stopping', runs its phases and only then sets 'stopped'. A phase that throws leaves + * the status at 'stopping' for good, so every later call returns without doing anything. + * A mock that simply completes the second time cannot show the bug this models. + */ +function fakeNode(body: () => Promise = async () => {}): { status: string; stop: () => Promise; calls: number } { + const node = { + status: 'started', + calls: 0, + stop: async (): Promise => { + node.calls++; + if (node.status !== 'started') return; + node.status = 'stopping'; + await body(); + node.status = 'stopped'; + }, + }; + return node; +} + +/** A promise plus the handles to settle it, so a test can hold a step open. */ +function deferred(): { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } { + let resolve!: (v: T) => void; + let reject!: (e: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +describe('Network lifecycle', () => { + it('a second start() while the first is still building does not build a second node', async () => { + const net = bareNetwork(); + const gate = deferred(); + let started = 0; + (net as any).startLocked = async (): Promise => { + started++; + await gate.promise; + }; + + const first = net.start([]); + // Second caller arrives while the first is parked inside startLocked — exactly the + // window in which `if (this.node)` was still false and let both through. + const second = net.start([]); + await Promise.resolve(); + expect(started).toBe(1); + + gate.resolve(); + await Promise.all([first, second]); + expect(started).toBe(1); + expect(net.isRunning()).toBe(true); + expect(net.getLifecycle()).toBe('running'); + }); + + it('a failed start leaves nothing running and does not block the next start', async () => { + const net = bareNetwork(); + let torndown = 0; + (net as any).teardown = async (): Promise => { + torndown++; + }; + (net as any).startLocked = async (): Promise => { + throw new Error('createLibp2p exploded'); + }; + + await expect(net.start([])).rejects.toThrow('createLibp2p exploded'); + expect(torndown).toBe(1); + expect(net.isRunning()).toBe(false); + expect(net.getLifecycle()).toBe('stopped'); + + // The whole point: a start that failed must be retryable in-process. + (net as any).startLocked = async (): Promise => {}; + await net.start([]); + expect(net.isRunning()).toBe(true); + }); + + it('start() is not reported as running until it has fully finished', async () => { + const net = bareNetwork(); + const gate = deferred(); + (net as any).startLocked = async (): Promise => { + // Whatever a partially built start has assigned, it is not running yet. + (net as any).node = { stop: async (): Promise => {} }; + await gate.promise; + }; + + const running = net.start([]); + await Promise.resolve(); + expect(net.isRunning()).toBe(false); + expect(net.getLifecycle()).toBe('starting'); + + gate.resolve(); + await running; + expect(net.isRunning()).toBe(true); + }); + + it('stop() waits for an in-flight start instead of tearing it down mid-build', async () => { + const net = bareNetwork(); + const gate = deferred(); + const order: string[] = []; + (net as any).startLocked = async (): Promise => { + order.push('start:begin'); + await gate.promise; + order.push('start:end'); + }; + (net as any).teardown = async (): Promise => { + order.push('teardown'); + }; + + const starting = net.start([]); + await Promise.resolve(); + const stopping = net.stop(); + // Give the stop every chance to cut in front of the unfinished start. + await Promise.resolve(); + await Promise.resolve(); + expect(order).toEqual(['start:begin']); + + gate.resolve(); + await Promise.all([starting, stopping]); + expect(order).toEqual(['start:begin', 'start:end', 'teardown']); + expect(net.getLifecycle()).toBe('stopped'); + }); + + it('a node that refuses to stop leaves the instance failed, not stopped', async () => { + const net = bareNetwork(); + let closed = 0; + const node = fakeNode(async () => { + throw new Error('node.stop failed'); + }); + (net as any).node = node; + (net as any).datastore = { + close: async (): Promise => { + closed++; + }, + }; + + await expect(net.stop()).rejects.toThrow('node.stop failed'); + + // The node may still hold its listener, its connections and its port. Closing the + // datastore under it and dropping the reference would make that both permanent and + // unretryable, and `stopped` would invite a second node over the same identity. + expect(closed).toBe(0); + expect((net as any).node).toBe(node); + expect((net as any).datastore).not.toBeNull(); + expect(net.getLifecycle()).toBe('failed'); + expect(net.isRunning()).toBe(false); + }); + + it('a failed stop refuses a new start and every destructive operation', async () => { + const net = bareNetwork(); + (net as any).node = fakeNode(async () => { + throw new Error('node.stop failed'); + }); + (net as any).datastore = { close: async (): Promise => {} }; + await expect(net.stop()).rejects.toThrow('node.stop failed'); + + let started = 0; + (net as any).startLocked = async (): Promise => { + started++; + }; + await expect(net.start([])).rejects.toThrow('failed state'); + expect(started).toBe(0); + + await expect(net.clearDatastore()).rejects.toThrow('Network must be stopped'); + await expect(net.clearPeerstore()).rejects.toThrow('Network must be stopped'); + await expect(net.clearIdentityKey()).rejects.toThrow('Network must be stopped'); + await expect(net.writeIdentityKey(new Uint8Array([1, 2, 3]))).rejects.toThrow('Network must be stopped'); + }); + + it('a stop that reaches "stopped" releases the node and the datastore', async () => { + const net = bareNetwork(); + let closed = 0; + const node = fakeNode(); + (net as any).node = node; + (net as any).datastore = { + close: async (): Promise => { + closed++; + }, + }; + + await net.stop(); + expect(node.status).toBe('stopped'); + expect(closed).toBe(1); + expect((net as any).node).toBeNull(); + expect((net as any).datastore).toBeNull(); + expect(net.getLifecycle()).toBe('stopped'); + }); + + it('a datastore that will not close leaves the instance failed, and the retry only closes it', async () => { + const net = bareNetwork(); + let attempts = 0; + const node = fakeNode(); + (net as any).node = node; + const datastore = { + close: async (): Promise => { + if (++attempts === 1) throw new Error('sqlite close failed'); + }, + }; + (net as any).datastore = datastore; + + await expect(net.stop()).rejects.toThrow('sqlite close failed'); + // Losing the reference here is what made the open handle unreachable AND let a wipe + // or a second start run over the database it still holds. + expect((net as any).datastore).toBe(datastore); + expect(net.getLifecycle()).toBe('failed'); + await expect(net.clearDatastore()).rejects.toThrow('Network must be stopped'); + + // The node was proved down on the first attempt and must not be stopped again — only + // the phase that is still outstanding repeats. + await net.stop(); + expect(attempts).toBe(2); + expect(node.calls).toBe(1); + expect((net as any).datastore).toBeNull(); + expect(net.getLifecycle()).toBe('stopped'); + }); + + it('an interrupted libp2p stop is terminal — a retry is refused, not faked', async () => { + const net = bareNetwork(); + let closed = 0; + const node = fakeNode(async () => { + throw new Error('transport close failed'); + }); + (net as any).node = node; + (net as any).datastore = { + close: async (): Promise => { + closed++; + }, + }; + + await expect(net.stop()).rejects.toThrow('transport close failed'); + expect(node.status).toBe('stopping'); + expect(net.getLifecycle()).toBe('failed'); + + // What a "retry" would actually reach: libp2p returns at once because the status is + // not 'started', having done nothing more. Reading that silent no-op as a successful + // shutdown is what handed back a node still holding its listener, port and connections. + await node.stop(); + expect(node.calls).toBe(2); + expect(node.status).toBe('stopping'); + + // So the wrapper refuses to try rather than report a shutdown it cannot perform. + await expect(net.stop()).rejects.toThrow('restart the process'); + expect(node.calls).toBe(2); + expect(closed).toBe(0); + expect((net as any).node).toBe(node); + expect((net as any).datastore).not.toBeNull(); + expect(net.getLifecycle()).toBe('failed'); + await expect(net.start([])).rejects.toThrow('failed state'); + }); + + it('a failed start whose cleanup also fails refuses the next start', async () => { + const net = bareNetwork(); + (net as any).teardown = async (): Promise => { + throw new Error('node.stop failed'); + }; + (net as any).startLocked = async (): Promise => { + throw new Error('createLibp2p exploded'); + }; + + // Both reasons survive: what broke the start, and why the instance is now unusable. + const err = await net.start([]).then( + () => null, + (e: unknown) => e + ); + expect(err).toBeInstanceOf(AggregateError); + expect((err as AggregateError).errors.map((e: unknown) => String((e as Error).message))).toEqual(['createLibp2p exploded', 'node.stop failed']); + expect(net.getLifecycle()).toBe('failed'); + + let started = 0; + (net as any).startLocked = async (): Promise => { + started++; + }; + await expect(net.start([])).rejects.toThrow('failed state'); + expect(started).toBe(0); + }); + + it('a start() that arrives while stop() is inside node.stop() is not answered "already running"', async () => { + const net = bareNetwork(); + const gate = deferred(); + let started = 0; + // The window in which `this.node` is still set but the run is over. + (net as any).node = fakeNode(() => gate.promise); + (net as any).startLocked = async (): Promise => { + started++; + }; + + const stopping = net.stop(); + await Promise.resolve(); + const starting = net.start([]); + gate.resolve(); + await Promise.all([stopping, starting]); + + // A pre-check on `this.node` outside the mutex would have logged "already + // running" and returned, leaving the caller with a stopped network. + expect(started).toBe(1); + expect(net.getLifecycle()).toBe('running'); + }); + + it('a datastore wipe cannot land inside an in-progress start', async () => { + const net = bareNetwork(); + const gate = deferred(); + let wiped = 0; + (net as any).startLocked = async (): Promise => { + // Mirrors the real start: the datastore is open and the identity read long + // before `this.node` is assigned. + await gate.promise; + }; + + const starting = net.start([]); + await Promise.resolve(); + expect(net.getLifecycle()).toBe('starting'); + + let refusal: unknown = null; + const wiping = net.clearDatastore().then( + () => { + wiped++; + }, + (err: unknown) => { + refusal = err; + } + ); + gate.resolve(); + await starting; + await wiping; + + // The wipe must be refused for the right reason. Guarding on `this.node` let it + // through here, because a start has no node yet while it has a datastore. + expect(wiped).toBe(0); + expect(String((refusal as Error)?.message)).toContain('Network must be stopped'); + expect(net.getLifecycle()).toBe('running'); + }); + + it('identity and peerstore writes are refused unless the lifecycle is stopped', async () => { + const net = bareNetwork(); + (net as any).startLocked = async (): Promise => {}; + await net.start([]); + + await expect(net.clearPeerstore()).rejects.toThrow('Network must be stopped'); + await expect(net.clearIdentityKey()).rejects.toThrow('Network must be stopped'); + await expect(net.writeIdentityKey(new Uint8Array([1, 2, 3]))).rejects.toThrow('Network must be stopped'); + }); +}); diff --git a/backend/tests/unit/protocol/network-mesh.test.ts b/backend/tests/unit/protocol/network-mesh.test.ts index 687ed4cab..f78de85e2 100644 --- a/backend/tests/unit/protocol/network-mesh.test.ts +++ b/backend/tests/unit/protocol/network-mesh.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'bun:test'; import { readFileSync } from 'fs'; import { join } from 'path'; import { LISH_TOPIC_PREFIX, DEFAULT_ACCEPT_PX_THRESHOLD, lishTopic, normalizeTrustedPeerIds, parseAcceptPXThreshold } from '../../../src/protocol/constants.ts'; +import { Network } from '../../../src/protocol/network.ts'; import { logStatusDebug } from '../../../src/protocol/status-logger.ts'; const NETWORK_TS = readFileSync(join(__dirname, '../../../src/protocol/network.ts'), 'utf-8'); @@ -348,25 +349,67 @@ describe('lishTopic helper', () => { // Peer count check scheduling — source code verification // --------------------------------------------------------------------------- -describe('subscribeTopic — peer count scheduling', () => { - it('schedules 3 delayed peer count checks after subscribe', () => { - const subscribeBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('subscribeTopic(networkID: string)'), NETWORK_TS.indexOf('unsubscribeHandler')); - const matches = subscribeBlock.match(/setTimeout\(\(\) => this\.schedulePeerCountCheck\(\)/g); - expect(matches).not.toBeNull(); - expect(matches!.length).toBe(3); +/** + * These used to grep the source for three literal setTimeout calls, which said nothing + * about what actually happens and broke the moment the calls were factored into a loop. + * The delayed probes matter for a different reason now: they used to be untracked, so + * they kept a closure on the instance alive and could fire against a node the run no + * longer owned. + */ +describe('subscribeTopic — delayed peer count probes', () => { + function bareNetwork() { + const checks: number[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).delayedPeerCountTimers = new Set(); + (network as any).schedulePeerCountCheck = () => checks.push(Date.now()); + return { network, checks }; + } + + it('tracks every probe it arms, so stop can cancel them', () => { + const { network } = bareNetwork(); + for (const delay of [2000, 5000, 15000]) (network as any).armDelayedPeerCountCheck(delay); + expect((network as any).delayedPeerCountTimers.size).toBe(3); + for (const timer of (network as any).delayedPeerCountTimers) clearTimeout(timer); }); - it('uses delays 2s, 5s, 15s for mesh rebuild', () => { + it('still uses the 2s, 5s and 15s mesh-rebuild delays', () => { const subscribeBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('subscribeTopic(networkID: string)'), NETWORK_TS.indexOf('unsubscribeHandler')); expect(subscribeBlock).toContain('2000'); expect(subscribeBlock).toContain('5000'); expect(subscribeBlock).toContain('15000'); }); + + it('does not check peer counts once the probe fires for a superseded run', async () => { + const { network, checks } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + (network as any).runEpoch = 2; // a stop()/start() landed while the probe was pending + await new Promise(resolve => setTimeout(resolve, 20)); + expect(checks).toEqual([]); + }); + + it('checks peer counts when the run is still the same', async () => { + const { network, checks } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + await new Promise(resolve => setTimeout(resolve, 20)); + expect(checks).toHaveLength(1); + }); + + it('forgets a timer once it has fired', async () => { + const { network } = bareNetwork(); + (network as any).armDelayedPeerCountCheck(1); + await new Promise(resolve => setTimeout(resolve, 20)); + expect((network as any).delayedPeerCountTimers.size).toBe(0); + }); }); describe('unsubscribeTopic — peer count scheduling', () => { it('calls schedulePeerCountCheck immediately', () => { - const unsubBlock = NETWORK_TS.slice(NETWORK_TS.indexOf('unsubscribeTopic(networkID: string)'), NETWORK_TS.indexOf('getTopicPeers')); + const unsubStart = NETWORK_TS.indexOf('unsubscribeTopic(networkID: string)'); + // Anchor the end marker AFTER the method start — getTopicPeers is now also + // referenced earlier (status-tick membership sweep), so a global indexOf + // would slice an empty range. + const unsubBlock = NETWORK_TS.slice(unsubStart, NETWORK_TS.indexOf('getTopicPeers', unsubStart)); expect(unsubBlock).toContain('this.schedulePeerCountCheck()'); }); }); diff --git a/backend/tests/unit/protocol/network-restart-safety.test.ts b/backend/tests/unit/protocol/network-restart-safety.test.ts new file mode 100644 index 000000000..10fc7f2d7 --- /dev/null +++ b/backend/tests/unit/protocol/network-restart-safety.test.ts @@ -0,0 +1,325 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * A destructive operation started on one libp2p node must never finish against the + * next one. Both paths below await in the middle, and both used to re-read + * `this.node` (or re-evaluate the run epoch) afterwards — so a stop()/start() + * landing in that window handed them the NEW node to hang up, purge and evict on. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const ADDR = `/ip4/192.0.2.10/tcp/9090/p2p/${PEER_ID}`; + +function deferred(): { promise: Promise; resolve: (v: T) => void } { + let resolve!: (v: T) => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +describe('Network.disconnectPeer — bound to the node it started on', () => { + function harness() { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map>(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + + const gate = deferred(); + const touchedByOld: string[] = []; + const nodeA = { + getConnections: (): unknown[] => [], + async hangUp(): Promise { + touchedByOld.push('A.hangUp'); + }, + peerStore: { + async merge(): Promise { + await gate.promise; + }, + async delete(): Promise { + touchedByOld.push('A.delete'); + }, + }, + }; + const touchedByNew: string[] = []; + const nodeB = { + getConnections: (): unknown[] => [], + async hangUp(): Promise { + touchedByNew.push('B.hangUp'); + }, + peerStore: { + async merge(): Promise {}, + async delete(): Promise { + touchedByNew.push('B.delete'); + }, + }, + }; + (network as any).node = nodeA; + return { network, nodeB, gate, touchedByOld, touchedByNew }; + } + + it('does not hang up or purge on the node that replaced it mid-flight', async () => { + const { network, nodeB, gate, touchedByNew } = harness(); + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + // stop() + start(): a new epoch and a new node, exactly as a restart leaves them. + (network as any).runEpoch = 2; + (network as any).node = nodeB; + gate.resolve(); + await leaving; + + expect(touchedByNew).toEqual([]); + }); + + it('does not purge on the new node when the restart lands during the hangUp', async () => { + const { network, nodeB, gate, touchedByNew } = harness(); + const hangUpGate = deferred(); + (network as any).node.hangUp = async (): Promise => { + await hangUpGate.promise; + }; + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + gate.resolve(); + await Promise.resolve(); + await Promise.resolve(); + (network as any).runEpoch = 2; + (network as any).node = nodeB; + hangUpGate.resolve(); + await leaving; + + expect(touchedByNew).toEqual([]); + }); + + it('still completes normally when no restart happens', async () => { + const { network, gate, touchedByOld } = harness(); + + const leaving = network.disconnectPeer(PEER_ID, 'net-a'); + await Promise.resolve(); + gate.resolve(); + await leaving; + + expect(touchedByOld).toEqual(['A.hangUp', 'A.delete']); + }); +}); + +describe('Network.addBootstrapPeers — a dial that lands after a restart', () => { + function harness() { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + // The peer is suppressed, so a dial landing on THIS run would legitimately take + // the destructive "landed after leave" branch. After a restart it must not. + (network as any).redialSuppressedByNet = new Map([['net-a', new Set([PEER_ID])]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending(): void {}, + recordOutcome(): void {}, + }; + const gate = deferred<{ remoteAddr: string }>(); + (network as any).node = { + peerId: { toString: (): string => 'selfID' }, + getConnections: (): unknown[] => [], + dial: (): Promise<{ remoteAddr: string }> => gate.promise, + peerStore: { async merge(): Promise {} }, + }; + const disconnected: string[] = []; + (network as any).disconnectPeer = async (pid: string): Promise => { + disconnected.push(pid); + }; + return { network, gate, disconnected }; + } + + it('does not disconnect on the new node when the epoch moved on', async () => { + const { network, gate, disconnected } = harness(); + + const dialing = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + (network as any).runEpoch = 2; + gate.resolve({ remoteAddr: ADDR }); + await dialing; + + expect(disconnected).toEqual([]); + }); + + it('a dial settling after a restart does not release the claim of the new run', async () => { + const { network } = harness(); + let dials = 0; + const gates: Array<{ promise: Promise<{ remoteAddr: string }>; resolve: (v: { remoteAddr: string }) => void }> = []; + (network as any).node.dial = (): Promise<{ remoteAddr: string }> => { + dials++; + let resolve!: (v: { remoteAddr: string }) => void; + const promise = new Promise<{ remoteAddr: string }>(res => { + resolve = res; + }); + gates.push({ promise, resolve }); + return promise; + }; + + // Run A claims the address on the old node. + const runA = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(1); + + // Teardown hands the next run a fresh claim table; run B takes the address in it. + (network as any).inFlightBootstrapDials = new Set(); + (network as any).runEpoch = 2; + const runB = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(2); + + // Run A finally settles. Its release must land in the table it claimed from. + gates[0]!.resolve({ remoteAddr: ADDR }); + await runA; + + // A third request for the same address must still find run B's claim in place. + const runC = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + expect(dials).toBe(2); + + gates[1]!.resolve({ remoteAddr: ADDR }); + await Promise.all([runB, runC]); + }); + + it('still disconnects when the dial lands on the same run after a leave', async () => { + const { network, gate, disconnected } = harness(); + // A leave bumps the generation; that must NOT short-circuit ahead of the + // disconnect — the connection this dial just opened is the thing to close. + (network as any).bootstrapGeneration = new Map([['net-a', 1]]); + + const dialing = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Promise.resolve(); + (network as any).bootstrapGeneration.set('net-a', 2); + gate.resolve({ remoteAddr: ADDR }); + await dialing; + + expect(disconnected).toEqual([PEER_ID]); + }); +}); + +/** + * Production starts the node with an empty bootstrap list, so the config-time + * `directPeers` seed never applies. A configured bootstrap therefore has to enter the + * gossipsub direct set on acceptance — waiting for the periodic promotion left the peer + * the whole mesh depends on PRUNE-able and without a fast reconnect for ~150 s. + */ +describe('Network.addBootstrapPeers — configured bootstraps become direct peers at once', () => { + function harness(suppressed: string[] = []) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending(): void {}, + recordOutcome(): void {}, + }; + const direct = new Set(); + (network as any).pubsub = { direct }; + (network as any).node = { + peerId: { toString: (): string => 'selfID' }, + getConnections: (): unknown[] => [], + dial: async (): Promise<{ remoteAddr: string }> => ({ remoteAddr: ADDR }), + peerStore: { async merge(): Promise {} }, + }; + (network as any).isPeerNeededByJoinedNetwork = (): boolean => false; + (network as any).isTopicSubscribed = (): boolean => true; + (network as any).rememberBootstrapAddress = (): void => {}; + return { network, direct }; + } + + it('adds a configured bootstrap on the dial that accepts it', async () => { + const { network, direct } = harness(); + + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + + expect([...direct]).toEqual([PEER_ID]); + }); + + it('leaves a merely discovered peer to the periodic promotion', async () => { + const { network, direct } = harness(); + + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + + expect([...direct]).toEqual([]); + }); +}); + +/** + * The configured-bootstrap lifecycle hands a peer two things when it answers a dial: a + * gossipsub `direct` entry (never PRUNEd, redialed every directConnectTicks) and a + * KEEP_ALIVE tag (libp2p redials it, the connection manager will not evict it). Removing + * the entry from the configuration used to take back neither, so the peer the user + * deleted went on being dialed for the rest of the run. + */ +describe('Network.pruneConfiguredBootstrapPeer — gives back what the entry was granted', () => { + function harness(neededByJoined: boolean) { + const network = Object.create(Network.prototype) as Network; + const direct = new Set(); + const merges: Array> = []; + (network as any).pubsub = { direct }; + (network as any).node = { + peerStore: { + async merge(_pid: unknown, data: Record): Promise { + merges.push(data); + }, + }, + }; + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = []; + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).addressProbeBackoff = new Map(); + (network as any).isPeerNeededByJoinedNetwork = (): boolean => neededByJoined; + (network as any).isRedialSuppressed = (): boolean => false; + (network as any).addGossipsubDirectPeer(PEER_ID); + return { network, direct, merges }; + } + + it('takes the direct entry and the keep-alive tag back', async () => { + const { network, direct, merges } = harness(false); + expect(direct.has(PEER_ID)).toBe(true); + + network.pruneConfiguredBootstrapPeer(PEER_ID); + // The tag removal is a fire-and-forget merge on the captured node. + for (let i = 0; i < 6; i++) await Promise.resolve(); + + expect(direct.has(PEER_ID)).toBe(false); + expect(merges).toEqual([{ tags: { 'keep-alive': undefined } }]); + }); + + it('leaves the keep-alive tag alone while a joined network still wants the peer', async () => { + const { network, direct, merges } = harness(true); + + network.pruneConfiguredBootstrapPeer(PEER_ID); + for (let i = 0; i < 6; i++) await Promise.resolve(); + + // The direct entry belongs to the bootstrap lifecycle and goes; the tag is now the + // joined network's, and the periodic promotion re-adds the direct entry if wanted. + expect(direct.has(PEER_ID)).toBe(false); + expect(merges).toEqual([]); + }); +}); diff --git a/backend/tests/unit/protocol/peer-announce.test.ts b/backend/tests/unit/protocol/peer-announce.test.ts index 03af65169..0345ce8ed 100644 --- a/backend/tests/unit/protocol/peer-announce.test.ts +++ b/backend/tests/unit/protocol/peer-announce.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'bun:test'; import { multiaddr as Multiaddr } from '@multiformats/multiaddr'; -import { PeerAnnounceManager, type PeerAnnounceMessage } from '../../../src/protocol/peer-announce.ts'; +import { AnnounceRateLimiter, PeerAnnounceManager, type PeerAnnounceMessage } from '../../../src/protocol/peer-announce.ts'; import { LISH_TOPIC_PREFIX } from '../../../src/protocol/constants.ts'; // Topic-scoping guard for peer-announce emit(): the transitive peer list broadcast @@ -229,3 +229,405 @@ describe('PeerAnnounceManager.emit recently-seen membership', () => { } }); }); + +// Inbound-intake guards. Every address that survives handle() costs a dial, a status +// row and a snapshot downstream, so the two things that bound that cost — collapsing +// addresses that mean the same thing, and capping what one announcing peer can spend — +// are asserted here rather than left to the receivers further down the chain. + +const SRC_ID = '12D3KooWSourceSourceSourceSourceSourceSourceSourceSS'; +const OTHER_SRC_ID = '12D3KooWOtherOtherOtherOtherOtherOtherOtherOtherOO'; +/** Identity every intake fixture terminates in — intake refuses addresses without one. */ +const ANNOUNCED_ID = '12D3KooWH3uVF6wv47WnArKHk5p6cvgCJEb74UTmxztmQDc298L3'; + +/** Append the announced identity, which inbound intake requires. */ +function withID(address: string, id: string = ANNOUNCED_ID): string { + return `${address}/p2p/${id}`; +} + +/** A manager wired only for handle(): captures the address lists it forwards. */ +function intakeManager() { + const forwarded: string[][] = []; + const mgr = new PeerAnnounceManager({ + getNode: () => null, + getPubsub: () => null, + broadcast: async () => {}, + addBootstrapPeers: async multiaddrs => { + forwarded.push(multiaddrs); + }, + }); + return { mgr, forwarded }; +} + +/** N distinct routable addresses in RFC5737 TEST-NET-3. */ +function distinctAddrs(count: number): string[] { + return Array.from({ length: count }, (_v, i) => withID(`/ip4/203.0.113.${i % 254}/tcp/${9000 + i}`)); +} + +describe('PeerAnnounceManager.handle address dedup', () => { + it('collapses one address repeated many times into a single entry', async () => { + const { mgr, forwarded } = intakeManager(); + const addr = withID('/ip4/198.51.100.7/tcp/9090'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: Array(300).fill(addr) }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[addr]]); + }); + + it('collapses two spellings of one address (DNS case, expanded vs compressed IPv6)', async () => { + const { mgr, forwarded } = intakeManager(); + const multiaddrs = [`/dns4/Peer.Example.COM/tcp/9090/p2p/${SELF_ID}`, `/dns4/peer.example.com/tcp/9090/p2p/${SELF_ID}`, withID('/ip6/2001:0db8:0000:0000:0000:0000:0000:0001/tcp/9090'), withID('/ip6/2001:db8::1/tcp/9090')]; + + await mgr.handle({ type: 'peer-announce', multiaddrs }, 'netAAAA', SRC_ID); + + // One DNS entry + one IPv6 entry, each keeping the spelling it arrived in. + expect(forwarded[0]).toEqual([multiaddrs[0]!, multiaddrs[2]!]); + }); + + it('counts UNIQUE addresses against the total cap, not raw entries', async () => { + // 300 copies of one address plus 5 distinct ones is 6 unique — the duplicates + // must not consume the 128-address budget the distinct ones need. + const { mgr, forwarded } = intakeManager(); + const dup = withID('/ip4/198.51.100.7/tcp/9090'); + const rest = distinctAddrs(5); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [...Array(300).fill(dup), ...rest] }, 'netAAAA', SRC_ID); + + expect(forwarded[0]).toEqual([dup, ...rest]); + }); + + it('still caps a flood of genuinely distinct addresses at 128', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: distinctAddrs(200) }, 'netAAAA', SRC_ID); + + expect(forwarded[0]!.length).toBe(128); + }); + + it('drops non-routable addresses before deduping', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [withID('/ip4/127.0.0.1/tcp/9090'), withID('/ip4/127.0.0.1/tcp/9090'), 'not-a-multiaddr'] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([]); + }); + + it('refuses an announced address that carries no /p2p identity', async () => { + // Without an identity the address is unreachable by every per-peer control + // downstream — backoff, quarantine, leave-suppression, purge — while one + // successful dial parks it on the recovery list for good. + const { mgr, forwarded } = intakeManager(); + const named = withID('/ip4/203.0.113.5/tcp/9090'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: ['/ip4/203.0.113.4/tcp/9090', named] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[named]]); + }); + + it('keeps the relay-circuit target identity, not the relay hop', async () => { + const { mgr, forwarded } = intakeManager(); + const relayed = `/ip4/203.0.113.6/tcp/9090/p2p/${SELF_ID}/p2p-circuit/p2p/${ANNOUNCED_ID}`; + + await mgr.handle({ type: 'peer-announce', multiaddrs: [relayed] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[relayed]]); + }); +}); + +describe('PeerAnnounceManager.handle per-source rate limit', () => { + it('lets a burst through, then throttles the same source', async () => { + // Budget is 384 addresses (3 × the 128 cap); a fourth full announce from the + // same source within the same second has nothing left to spend. + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + + expect(forwarded.map(f => f.length)).toEqual([128, 128, 128]); // 4th announce dropped entirely + }); + + it('throttles one source without starving another', async () => { + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + forwarded.length = 0; + await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', OTHER_SRC_ID); + + expect(forwarded.map(f => f.length)).toEqual([128]); + }); +}); + +describe('AnnounceRateLimiter', () => { + it('grants a full burst to an unseen source and nothing more', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + expect(limiter.take('a', 384, 0)).toBe(384); + expect(limiter.take('a', 1, 0)).toBe(0); + }); + + it('grants partially rather than refusing outright when over budget', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + expect(limiter.take('a', 500, 0)).toBe(384); + }); + + it('recovers over time at the configured rate', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('a', 200, 30_000)).toBe(128); // half a minute → half the per-minute rate + expect(limiter.take('a', 200, 60_000)).toBe(128); // another 30s worth + }); + + it('never refills past the burst ceiling', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('a', 1000, 600_000)).toBe(384); // ten idle minutes still caps at burst + }); + + it('keeps one source spending from starving another', () => { + const limiter = new AnnounceRateLimiter(384, 256, 1024); + limiter.take('a', 384, 0); + expect(limiter.take('b', 384, 0)).toBe(384); + }); + + it('bounds the bucket table, evicting the least recently heard-from source', () => { + const limiter = new AnnounceRateLimiter(10, 10, 2); + limiter.take('a', 10, 0); // 'a' exhausted, then pushed out by 'c' + limiter.take('b', 10, 0); + limiter.take('c', 10, 0); + + expect(limiter.take('a', 10, 0)).toBe(10); // evicted → fresh bucket + expect(limiter.take('c', 1, 0)).toBe(0); // recently used → still exhausted + }); +}); + +/** + * Both maps belong to the run that filled them. Membership is recorded against the libp2p + * node that was up at the time and is what leave-network reads to decide who to hang up; + * the rate-limiter buckets are per-source budgets. Carrying either into the next start() + * makes the new run act on the old one's state. + */ +describe('PeerAnnounceManager.stop clears per-run state', () => { + const TOPIC = `${LISH_TOPIC_PREFIX}netAAAA`; + + it('forgets topic membership', () => { + const { mgr } = intakeManager(); + mgr.noteMember(TOPIC, PA_ID); + expect(mgr.getRecentMembers(TOPIC)).toEqual([PA_ID]); + + mgr.stop(); + + expect(mgr.getRecentMembers(TOPIC)).toEqual([]); + }); + + it('gives a throttled source its full budget back after a restart', async () => { + const { mgr, forwarded } = intakeManager(); + const addrs = distinctAddrs(128); + for (let i = 0; i < 4; i++) await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + expect(forwarded).toHaveLength(3); // the fourth was over budget + + mgr.stop(); + mgr.start(); + await mgr.handle({ type: 'peer-announce', multiaddrs: addrs }, 'netAAAA', SRC_ID); + + expect(forwarded).toHaveLength(4); + mgr.stop(); + }); +}); + +/** + * The unique cap bounds what intake ADMITS, not what the walk costs. Every raw entry is + * parsed, canonicalised and routability-tested before dedup can discard it, so a message + * of thousands of duplicates or junk values bought that work unbounded. + */ +describe('PeerAnnounceManager.handle raw input bound', () => { + it('stops examining a flood of duplicates long before the end of the list', async () => { + // The unique address sits past the raw budget, so reaching it would mean the walk + // went all the way through the padding. + const { mgr, forwarded } = intakeManager(); + const dup = withID('/ip4/198.51.100.7/tcp/9090'); + const beyond = withID('/ip4/203.0.113.200/tcp/9099'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [...Array(5000).fill(dup), beyond] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[dup]]); + }); + + it('still admits a legitimate full-size announce', async () => { + const { mgr, forwarded } = intakeManager(); + + await mgr.handle({ type: 'peer-announce', multiaddrs: distinctAddrs(128) }, 'netAAAA', SRC_ID); + + expect(forwarded[0]!.length).toBe(128); + }); + + it('drops an over-long value without parsing it', async () => { + const { mgr, forwarded } = intakeManager(); + const bloated = `/dns4/${'a'.repeat(600)}.example.com/tcp/9090/p2p/${ANNOUNCED_ID}`; + const named = withID('/ip4/203.0.113.5/tcp/9090'); + + await mgr.handle({ type: 'peer-announce', multiaddrs: [bloated, named] }, 'netAAAA', SRC_ID); + + expect(forwarded).toEqual([[named]]); + }); +}); + +/** + * The emitter's lifecycle is a boolean plus a single timer handle, and scheduleNext() + * awaits peerStore.all() before arming that timer. A loop parked in that await across + * a stop() AND a start() used to find the flag reset and carry on, so two loops armed + * timers into one field and the next stop() cancelled only one of them. + */ +describe('PeerAnnounceManager lifecycle — one loop per start', () => { + function gatedNode(gate: Promise) { + return { + peerId: { toString: (): string => SELF_ID }, + getMultiaddrs: (): unknown[] => [Multiaddr(SELF_ADDR)], + peerStore: { + all: async (): Promise => { + await gate; + return peersWithFillers(); + }, + }, + }; + } + + it('a loop parked across a stop/start does not arm a second timer', async () => { + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [] }; + const { mgr } = buildManager(gatedNode(gate), pubsub); + + const realSetTimeout = globalThis.setTimeout; + let armed = 0; + globalThis.setTimeout = ((fn: () => void, ms?: number) => { + armed++; + return realSetTimeout(fn, ms); + }) as typeof globalThis.setTimeout; + try { + mgr.start(); // loop A parks in peerStore.all() + await Promise.resolve(); + mgr.stop(); + mgr.start(); // loop B parks in its own peerStore.all() + release(); + // Let both awaits settle. + for (let i = 0; i < 8; i++) await Promise.resolve(); + expect(armed).toBe(1); + } finally { + globalThis.setTimeout = realSetTimeout; + mgr.stop(); + } + }); + + it('an emit whose run ended mid-await publishes nothing', async () => { + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + const { mgr, broadcasts } = buildManager(gatedNode(gate), pubsub); + + mgr.start(); + const emitting = (mgr as any).emit((mgr as any).generation); + mgr.stop(); + release(); + await emitting; + + expect(broadcasts).toEqual([]); + }); + + it('a publish parked across a stop/start never reaches the next topic', async () => { + // Two topics, so the emit has a second awaited publish after the first one returns. + let release!: () => void; + const held = new Promise(res => { + release = res; + }); + const oldPubsub = { getTopics: (): string[] => [TOPIC_A, TOPIC_B], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + const newPubsub = { getTopics: (): string[] => [TOPIC_A, TOPIC_B], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + let pubsub: any = oldPubsub; + const seen: Array<{ topic: string; onNew: boolean }> = []; + const mgr = new PeerAnnounceManager({ + getNode: () => gatedNode(Promise.resolve()) as any, + getPubsub: () => pubsub, + broadcast: async (topic, _msg, target) => { + seen.push({ topic, onNew: target === newPubsub }); + if (topic === TOPIC_A) await held; + }, + addBootstrapPeers: async () => {}, + }); + + mgr.start(); + const emitting = (mgr as any).emit((mgr as any).generation); + // Let the emit reach the first publish and park there. + for (let i = 0; i < 8; i++) await Promise.resolve(); + expect(seen).toHaveLength(1); + + // The run ends and a new one begins while topic A's publish is still outstanding. + mgr.stop(); + pubsub = newPubsub; + mgr.start(); + release(); + await emitting; + mgr.stop(); + + // Topic B belongs to the new run; the old emit must not have published it at all, + // and nothing it did publish may have gone over the new transport. + expect(seen.map(s => s.topic)).toEqual([TOPIC_A]); + expect(seen.some(s => s.onNew)).toBe(false); + }); + + it('an uninterrupted emit still publishes', async () => { + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + const { mgr, broadcasts } = buildManager(gatedNode(Promise.resolve()), pubsub); + + mgr.start(); + await (mgr as any).emit((mgr as any).generation); + mgr.stop(); + + expect(broadcasts).toHaveLength(1); + }); +}); + +/** + * A peerStore entry for A can hold an address that terminates in /p2p/B — stale config, + * a poisoned announce, or a peer that changed identity. Broadcasting it verbatim taught + * every receiver that the address belongs to B. + */ +describe('PeerAnnounceManager.emit — transitive addresses must name their own peer', () => { + function emitWith(addresses: string[]) { + const peer = { id: { toString: (): string => PA_ID }, addresses: addresses.map(a => ({ multiaddr: Multiaddr(a) })) }; + const allPeers = peersWithFillers(peer as any); + const node = { + peerId: { toString: (): string => SELF_ID }, + getMultiaddrs: (): unknown[] => [Multiaddr(SELF_ADDR)], + peerStore: { all: async (): Promise => allPeers }, + }; + const pubsub = { getTopics: (): string[] => [TOPIC_A], getSubscribers: (): unknown[] => [fakeSubscriber(PA_ID)] }; + return buildManager(node, pubsub); + } + + it('drops an address of one peer that ends in another peer identity', async () => { + const { mgr, broadcasts } = emitWith([`${PA_ADDR}/p2p/${PB_ID}`]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toEqual([SELF_ADDR]); + }); + + it('keeps an address that already ends in the right identity', async () => { + const { mgr, broadcasts } = emitWith([`${PA_ADDR}/p2p/${PA_ID}`]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toContain(`${PA_ADDR}/p2p/${PA_ID}`); + }); + + it('appends the identity to a bare address', async () => { + const { mgr, broadcasts } = emitWith([PA_ADDR]); + + await (mgr as any).emit(); + + expect(broadcasts[0]!.msg.multiaddrs).toContain(`${PA_ADDR}/p2p/${PA_ID}`); + }); +}); diff --git a/backend/tests/unit/protocol/peer-discovery-handler.test.ts b/backend/tests/unit/protocol/peer-discovery-handler.test.ts new file mode 100644 index 000000000..6fd6ee17f --- /dev/null +++ b/backend/tests/unit/protocol/peer-discovery-handler.test.ts @@ -0,0 +1,335 @@ +import { describe, it, expect } from 'bun:test'; +import { Network } from '../../../src/protocol/network.ts'; + +/** + * The `peer:discovery` handler. libp2p delivers these events from mDNS, identify and + * gossipsub PX alike, including for peers this node has already written off — so the + * handler is a dial path like any other and has to respect the same pacing. It used to + * respect only leave-network suppression, and it stamped the `keep-alive-fleet` re-dial + * instruction before making any contact at all. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +const ADDR = { toString: () => `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}` }; + +type Handler = (evt: any) => void | Promise; + +function bareNetwork(opts: { connected?: boolean; dialFails?: boolean } = {}) { + const dialled: unknown[][] = []; + const tagged: string[] = []; + const handlers = new Map(); + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).listeners = []; + (network as any).redialSuppressedByNet = new Map(); + (network as any).inFlightDiscoveryDials = new Set(); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).dcutrPeers = new Set(); + (network as any).peerDisconnectHandlers = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).recentDisconnects = []; + (network as any).lastMeshChange = new Map(); + (network as any).pubsub = null; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + addEventListener(event: string, handler: Handler) { + handlers.set(event, handler); + }, + getConnections: () => (opts.connected ? [{}] : []), + getPeers: () => [], + async dial(multiaddrs: unknown[]): Promise { + dialled.push(multiaddrs); + if (opts.dialFails) throw new Error('dial timed out'); + return {}; + }, + peerStore: { + async merge(_pid: unknown, patch: { tags?: Record }): Promise { + for (const key of Object.keys(patch.tags ?? {})) tagged.push(key); + }, + }, + }; + (network as any).setupEventListeners(); + const fire = async (): Promise => { + await handlers.get('peer:discovery')!({ detail: { id: { toString: () => PEER_ID }, multiaddrs: [ADDR] } }); + }; + return { network, dialled, tagged, fire }; +} + +describe('peer:discovery — pacing', () => { + it('dials a peer nothing is holding back', async () => { + const { dialled, fire } = bareNetwork(); + await fire(); + expect(dialled).toHaveLength(1); + }); + + /** + * The eviction path quarantines a peer precisely so it stops being re-dialed. A late + * discovery event arriving right after must not undo that. + */ + it('refuses to dial a quarantined peer', async () => { + const { network, dialled, fire } = bareNetwork(); + (network as any).unreachableQuarantine.set(PEER_ID, Date.now()); + await fire(); + expect(dialled).toEqual([]); + }); + + it('refuses to dial a peer inside its redial backoff', async () => { + const { network, dialled, fire } = bareNetwork(); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 2, firstFailure: Date.now(), evictionFails: 0 }); + await fire(); + expect(dialled).toEqual([]); + }); + + it('records a failed discovery dial into the backoff', async () => { + const { network, fire } = bareNetwork({ dialFails: true }); + await fire(); + expect((network as any).redialBackoff.get(PEER_ID)?.nextAttempt).toBeGreaterThan(Date.now()); + }); +}); + +describe('peer:discovery — keep-alive tagging', () => { + /** + * The tag is a standing instruction to libp2p's ReconnectQueue. Writing it off an + * unverified claim is what let an evicted peer get its re-dial instruction back. + */ + it('does not tag a peer whose dial failed', async () => { + const { tagged, fire } = bareNetwork({ dialFails: true }); + await fire(); + expect(tagged).toEqual([]); + }); + + it('does not tag a quarantined peer it never contacted', async () => { + const { network, tagged, fire } = bareNetwork(); + (network as any).unreachableQuarantine.set(PEER_ID, Date.now()); + await fire(); + expect(tagged).toEqual([]); + }); + + it('tags a peer once the dial succeeds', async () => { + const { tagged, fire } = bareNetwork(); + await fire(); + expect(tagged).toEqual(['keep-alive-fleet']); + }); + + it('tags a peer we are already connected to, without dialing again', async () => { + const { dialled, tagged, fire } = bareNetwork({ connected: true }); + await fire(); + expect(tagged).toEqual(['keep-alive-fleet']); + expect(dialled).toEqual([]); + }); +}); + +/** + * leave-network and discovery racing each other. `disconnectPeer` yields twice before it + * finishes, and a discovery event landing in that window used to read "not suppressed", + * start a dial, and complete it after the hangUp had already found nothing to close — + * leaving the peer connected with the leave apparently done. + */ +describe('peer:discovery — a dial that lands after leave-network', () => { + function racingNetwork(resumeDialInsideLeave = false) { + const hungUp: string[] = []; + const tagged: string[] = []; + const handlers = new Map(); + let releaseDial: () => void = () => {}; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).listeners = []; + (network as any).redialSuppressedByNet = new Map(); + (network as any).inFlightDiscoveryDials = new Set(); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).dcutrPeers = new Set(); + (network as any).peerDisconnectHandlers = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).recentDisconnects = []; + (network as any).lastMeshChange = new Map(); + (network as any).pubsub = { getTopics: () => [] }; + (network as any).peerAnnounce = { getRecentMembers: () => [] }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + addEventListener(event: string, handler: Handler) { + handlers.set(event, handler); + }, + getConnections: () => [], + getPeers: () => [], + async dial(): Promise { + await new Promise(resolve => { + releaseDial = resolve; + }); + return {}; + }, + async hangUp(pid: { toString(): string }): Promise { + hungUp.push(pid.toString()); + }, + peerStore: { + async merge(_pid: unknown, patch: { tags?: Record }): Promise { + // Only STAMPS count — leave-network merges the same keys with `undefined` + // to remove them, and that is the opposite of what these tests watch for. + const removing = Object.values(patch.tags ?? {}).some(value => value === undefined); + for (const [key, value] of Object.entries(patch.tags ?? {})) if (value !== undefined) tagged.push(key); + // This merge is disconnectPeer's first await. Letting the parked discovery + // dial finish HERE is the actual race: the handler resumes before hangUp + // has run, which is the window the ordering fix has to cover. + if (removing && resumeDialInsideLeave) { + releaseDial(); + await Bun.sleep(0); + } + }, + async delete(): Promise {}, + }, + }; + (network as any).setupEventListeners(); + const fire = (): Promise => handlers.get('peer:discovery')!({ detail: { id: { toString: () => PEER_ID }, multiaddrs: [ADDR] } }) as Promise; + return { network, hungUp, tagged, fire, releaseDial: (): void => releaseDial() }; + } + + it('claims the suppression before disconnectPeer yields', async () => { + const { network } = racingNetwork(); + const leaving = (network as any).disconnectPeer(PEER_ID, 'net-a'); + expect((network as any).redialSuppressedByNet.get('net-a')?.has(PEER_ID)).toBe(true); + await leaving; + }); + + it('closes a discovery dial that completed after the peer was left', async () => { + const { network, hungUp, tagged, fire, releaseDial } = racingNetwork(); + const discovering = fire(); // parks inside dial() + await Bun.sleep(1); + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + hungUp.length = 0; // the leave's own hangUp found nothing; watch what happens next + releaseDial(); + await discovering; + + expect(hungUp).toEqual([PEER_ID]); + expect(tagged).toEqual([]); // and never re-armed the re-dial instruction + }); + + it('keeps a late dial whose peer another joined network still needs', async () => { + const { network, hungUp, fire, releaseDial } = racingNetwork(); + const discovering = fire(); + await Bun.sleep(1); + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + (network as any).configuredBootstrapPeerIDs.add(PEER_ID); // infrastructure elsewhere + hungUp.length = 0; + releaseDial(); + await discovering; + + expect(hungUp).toEqual([]); + }); + + /** + * The interleaving itself: the discovery dial completes DURING `disconnectPeer`, after + * the tag removal and before the hangUp. Nothing later in the leave will close that + * connection, so the handler has to see the suppression already recorded. + */ + it('closes a dial that completes inside the leave, before its hangUp runs', async () => { + const { network, hungUp, tagged, fire } = racingNetwork(true); + const discovering = fire(); + await Bun.sleep(1); + + await (network as any).disconnectPeer(PEER_ID, 'net-a'); + await discovering; + + expect(tagged).toEqual([]); + expect(hungUp).toContain(PEER_ID); + }); +}); + +/** + * mDNS, identify and PX all raise a discovery event for the same arrival. The per-peer + * backoff cannot separate them — it is written only once a dial has already FAILED — so + * without a claim every event started its own dial of the same identity. + */ +describe('peer:discovery — one dial per peer at a time', () => { + it('collapses concurrent events for one peer into a single dial', async () => { + const { network, dialled, fire } = bareNetwork(); + let release!: () => void; + const gate = new Promise(res => { + release = res; + }); + (network as any).node.dial = async (multiaddrs: unknown[]): Promise => { + dialled.push(multiaddrs); + await gate; + return {}; + }; + + const first = fire(); + const second = fire(); + const third = fire(); + expect(dialled).toHaveLength(1); + + release(); + await Promise.all([first, second, third]); + + // Claim released — a later event is free to dial again. + await fire(); + expect(dialled).toHaveLength(2); + }); +}); + +/** + * The addresses of every discovery event are already in the peerStore by the time this + * handler runs: libp2p merges each discovery service's list in `#onDiscoveryPeer` and + * only then dispatches the public `peer:discovery`. A skipped duplicate therefore loses + * nothing, and an application-level merge would only be an unguarded late write — one + * that can land after a leave, an eviction or a stop and put back what those removed. + */ +describe('peer:discovery — a skipped duplicate writes nothing of its own', () => { + const RELAY_ADDR = { toString: () => `/ip4/203.0.113.9/tcp/9090/p2p/RelayXYZ/p2p-circuit/p2p/${PEER_ID}` }; + const DIRECT_ADDR = { toString: () => `/ip4/198.51.100.4/tcp/9090/p2p/${PEER_ID}` }; + + it('does not merge addresses of its own while a dial for the peer is in flight', async () => { + const handlers = new Map(); + const merged: unknown[][] = []; + const dialled: unknown[][] = []; + let releaseDial: () => void = () => {}; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).listeners = []; + (network as any).redialSuppressedByNet = new Map(); + (network as any).inFlightDiscoveryDials = new Set(); + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).dcutrPeers = new Set(); + (network as any).peerDisconnectHandlers = new Set(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).recentDisconnects = []; + (network as any).lastMeshChange = new Map(); + (network as any).pubsub = null; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + addEventListener(event: string, handler: Handler) { + handlers.set(event, handler); + }, + getConnections: () => [], + getPeers: () => [], + async dial(multiaddrs: unknown[]): Promise { + dialled.push(multiaddrs); + await new Promise(resolve => { + releaseDial = resolve; + }); + return {}; + }, + peerStore: { + async merge(_pid: unknown, patch: { multiaddrs?: unknown[] }): Promise { + if (patch.multiaddrs) merged.push(patch.multiaddrs); + }, + }, + }; + (network as any).setupEventListeners(); + const fire = async (addr: unknown): Promise => await handlers.get('peer:discovery')!({ detail: { id: { toString: () => PEER_ID }, multiaddrs: [addr] } }); + + const first = fire(RELAY_ADDR); + for (let i = 0; i < 4; i++) await Promise.resolve(); + // Second source names the same peer on a better address while the first dial hangs. + await fire(DIRECT_ADDR); + + expect(dialled).toHaveLength(1); + expect(merged).toEqual([]); + + releaseDial(); + await first; + }); +}); diff --git a/backend/tests/unit/protocol/peer-eviction-guards.test.ts b/backend/tests/unit/protocol/peer-eviction-guards.test.ts new file mode 100644 index 000000000..2afda09e0 --- /dev/null +++ b/backend/tests/unit/protocol/peer-eviction-guards.test.ts @@ -0,0 +1,2070 @@ +import { describe, it, expect } from 'bun:test'; +import { multiaddr } from '@multiformats/multiaddr'; +import { Mutex } from 'async-mutex'; +import { Network, isRecoveryDialDue, isSameDialEndpoint, normalizeMultiaddrForCompare } from '../../../src/protocol/network.ts'; +import { BootstrapStatusTracker } from '../../../src/protocol/bootstrap-status.ts'; + +/** + * Guards on the DESTRUCTIVE peer-eviction paths. The pure decision helpers are covered + * in peer-eviction.test.ts; what is exercised here is the part that actually closes + * connections and deletes peerStore entries, where the damage from getting it wrong is + * not a wrong boolean but a live peer evicted from the wrong libp2p node. + */ + +const PEER_ID = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; +/** + * A peerStore entry with no addresses at all is the deterministic form of "nothing the + * dial gater will let us try" — using a real address would make the test depend on + * which subnets this host happens to be on. + */ +const NO_ADDRESSES: Array<{ multiaddr: { toString(): string } }> = []; + +function peerIdLike(id: string) { + return { toString: () => id, equals: (o: any) => String(o) === id }; +} + +describe('purgeStalePeer — epoch guard', () => { + function bareNetwork(onClose?: () => void) { + const deleted: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).node = { + getConnections: () => [ + { + async close(): Promise { + onClose?.(); + }, + }, + ], + peerStore: { + async delete(pid: { toString(): string }): Promise { + deleted.push(pid.toString()); + }, + async merge(): Promise {}, + }, + }; + return { network, deleted }; + } + + it('deletes the peerStore entry while the run still owns the node', async () => { + const { network, deleted } = bareNetwork(); + await (network as any).purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([PEER_ID]); + }); + + it('does not touch the peerStore when stop() lands while connections are closing', async () => { + // The exact race the epoch counter exists for: stop()/start() swaps in a new + // node during the close await, and the old run must not delete a peer from it. + const { network, deleted } = bareNetwork(); + const net = network as any; + net.node.getConnections = () => [ + { + async close(): Promise { + net.runEpoch++; // stop() during the await + }, + }, + ]; + await net.purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([]); + }); + + it('refuses to start at all for a stale epoch', async () => { + const { network, deleted } = bareNetwork(); + (network as any).runEpoch = 2; + await (network as any).purgeStalePeer(PEER_ID, 'test', 1); + expect(deleted).toEqual([]); + }); +}); + +/** + * A peer whose every stored address is rejected by the dial gater is not proof the peer + * is gone — a LAN/VPN-only peer looks exactly like this the moment our own interface + * drops. This path must therefore take the same self-online evidence as the + * dial-failure path, or a local outage evicts the whole non-configured peerStore. + */ +describe('runRedialMaintenance — eviction with no reachable address', () => { + function bareNetwork(opts: { weAreOnline: boolean; sinceMsAgo: number; configured?: boolean }) { + const purged: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - opts.sinceMsAgo]]); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { getConnections: () => [] }; + // Only the peer itself is ever asked about; "online" means we hold a connection + // to somebody else. + (network as any).hasConnectionOtherThan = () => opts.weAreOnline; + (network as any).purgeStalePeer = async (pid: string): Promise => { + purged.push(pid); + }; + return { network, purged }; + } + + const undialablePeer = { id: peerIdLike(PEER_ID), addresses: NO_ADDRESSES }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [undialablePeer], 1); + + it('evicts once the window has passed and we are demonstrably online', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000 }); + await run(network); + expect(purged).toEqual([PEER_ID]); + }); + + it('keeps the peer when the outage is ours', async () => { + const { network, purged } = bareNetwork({ weAreOnline: false, sinceMsAgo: 45 * 60_000 }); + await run(network); + expect(purged).toEqual([]); + }); + + it('slides the window forward during our outage instead of accumulating', async () => { + // Otherwise the first tick after connectivity returns would evict everything + // that went unreachable while we were down. + const { network } = bareNetwork({ weAreOnline: false, sinceMsAgo: 45 * 60_000 }); + await run(network); + const since = (network as any).noReachableSince.get(PEER_ID) as number; + expect(Date.now() - since).toBeLessThan(60_000); + }); + + it('never evicts a configured peer', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000, configured: true }); + await run(network); + expect(purged).toEqual([]); + }); + + it('keeps a peer that reconnected while the window was running', async () => { + const { network, purged } = bareNetwork({ weAreOnline: true, sinceMsAgo: 45 * 60_000 }); + (network as any).node = { getConnections: () => [{}] }; // live again + await run(network); + expect(purged).toEqual([]); + }); +}); + +/** + * An address may enter the address book only once libp2p has actually connected over + * it. Deciding that up-front from "do we have any connection to this peer" was too + * coarse — it skipped relay→direct upgrades and left bad addresses untested — so the + * answer is now read off the connection libp2p returns. + */ +describe('addBootstrapPeers — only a verified address enters the peerStore', () => { + function bareNetwork(remoteAddrOfReturnedConn: string) { + const merges: Array> = []; + const forced: boolean[] = []; + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + const outcomes: string[] = []; + const actualPeerIDs: Array = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string, _msg: unknown, actualPeerID: string | null) { + outcomes.push(status); + actualPeerIDs.push(actualPeerID); + }, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }, opts?: { force?: boolean }): Promise { + dialled.push(ma.toString()); + forced.push(opts?.force === true); + return { remoteAddr: { toString: () => remoteAddrOfReturnedConn }, remotePeer: peerIdLike(PEER_ID) }; + }, + peerStore: { + async merge(_pid: unknown, patch: Record): Promise { + merges.push(patch); + }, + }, + }; + return { network, merges, dialled, forced, outcomes, actualPeerIDs }; + } + + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + it('dials the address even when the peer is already connected', async () => { + // libp2p reuses a suitable connection by itself and dials when the new address + // would upgrade a relayed one; pre-empting that lost the upgrade. + const { network, dialled } = bareNetwork(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`); + (network as any).node.getConnections = () => [{}]; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR]); + }); + + it('stores the address when the connection is actually on it', async () => { + const { network, merges } = bareNetwork(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(merges).toHaveLength(1); + expect(merges[0]).toHaveProperty('multiaddrs'); + }); + + /** + * `force: true` defeats connection reuse but not a dial to the same peer id already + * in libp2p's queue — this call joins that job and can be handed the connection its + * OTHER address won. A configured row means "this address works", so it must not go + * green on a connection that never touched it. + */ + it('leaves a configured row pending when the connection came back on another address', async () => { + const { network, outcomes } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(outcomes).toEqual([]); + }); + + it('still records a discovered row, whose status only claims the peer answered', async () => { + const { network, outcomes } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(outcomes).toEqual(['connected']); + }); + + it('records the configured row once the connection is on the address itself', async () => { + const { network, outcomes } = bareNetwork(ADDR); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(outcomes).toEqual(['connected']); + }); + + /** + * The row-cap ranking only protects a peer whose identity we have actually PROVEN, and + * the successful dial is the only place that proof exists. Recording null there meant + * the production path never produced a protected row at all — the protection was real + * only in tests that wrote the field by hand. + */ + it('records the identity Noise proved on the connection as the verified peer ID', async () => { + const { network, actualPeerIDs } = bareNetwork(ADDR); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(actualPeerIDs).toEqual([PEER_ID]); + }); + + it('withholds the address when libp2p answered over a different one', async () => { + // Reused/relayed connection: this address was never contacted, so it is not + // Noise-verified and must not be poisonable into the address book. + const { network, merges } = bareNetwork(`/ip4/198.51.100.1/tcp/4001/p2p-circuit/p2p/${PEER_ID}`); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(merges).toHaveLength(1); + expect(merges[0]).not.toHaveProperty('multiaddrs'); + }); +}); + +/** + * The address-equality test behind "was THIS address verified". It decides whether an + * unverified address may enter the peerStore, so a false positive is a security bug, + * not a cosmetic one. + */ +describe('isSameDialEndpoint', () => { + const PEER_A = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + + it('accepts the same endpoint with and without the peer id suffix', () => { + expect(isSameDialEndpoint('/ip4/203.0.113.4/tcp/9090', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(true); + }); + + it('rejects a different port that merely shares a prefix', () => { + // /tcp/80 is a string prefix of /tcp/8080 — prefix matching would call a + // connection on 8080 proof that the claimed port 80 works. + expect(isSameDialEndpoint(`/ip4/203.0.113.4/tcp/8080/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/80/p2p/${PEER_A}`)).toBe(false); + }); + + it('rejects a different host', () => { + expect(isSameDialEndpoint(`/ip4/203.0.113.5/tcp/9090/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); + + it('rejects a relayed connection as proof of a direct address', () => { + expect(isSameDialEndpoint(`/ip4/198.51.100.1/tcp/4001/p2p/${PEER_A}/p2p-circuit/p2p/${PEER_A}`, `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); + + it('ignores DNS case and a trailing dot', () => { + expect(isSameDialEndpoint('/dns4/EXAMPLE.COM./tcp/443', '/dns4/example.com/tcp/443')).toBe(true); + }); + + it('treats a missing connection address as no proof', () => { + expect(isSameDialEndpoint('', `/ip4/203.0.113.4/tcp/9090/p2p/${PEER_A}`)).toBe(false); + }); + + it('rejects a relay whose identity differs only in case', () => { + // Only the trailing /p2p/ is stripped, so a circuit address is compared with + // the RELAY's peer id still in the middle. Folding the whole string would make two + // distinct relays look like one endpoint. + const relay = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + const otherRelay = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrnGS17fo'; + expect(isSameDialEndpoint(`/ip4/198.51.100.1/tcp/4001/p2p/${relay}/p2p-circuit/p2p/${PEER_A}`, `/ip4/198.51.100.1/tcp/4001/p2p/${otherRelay}/p2p-circuit/p2p/${PEER_A}`)).toBe(false); + }); +}); + +/** + * Case folding is a hostname question, not an identifier question. DNS is defined as + * case-insensitive; a base58 peer id is not, and the two live in the same string. + */ +describe('normalizeMultiaddrForCompare', () => { + it('folds DNS host case and drops the FQDN root dot', () => { + expect(normalizeMultiaddrForCompare('/dns4/EXAMPLE.COM./tcp/443')).toBe('/dns4/example.com/tcp/443'); + }); + + it('folds every DNS protocol variant', () => { + expect(normalizeMultiaddrForCompare('/dnsaddr/Bootstrap.Example.COM/tcp/443')).toBe('/dnsaddr/bootstrap.example.com/tcp/443'); + expect(normalizeMultiaddrForCompare('/dns6/Example.COM/tcp/443')).toBe('/dns6/example.com/tcp/443'); + }); + + it('leaves a peer id untouched', () => { + const peer = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + expect(normalizeMultiaddrForCompare(`/dns4/EXAMPLE.COM/tcp/443/p2p/${peer}`)).toBe(`/dns4/example.com/tcp/443/p2p/${peer}`); + }); + + it('leaves an address with no DNS component completely alone', () => { + const addr = '/ip4/203.0.113.4/tcp/9090/p2p/12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fo'; + expect(normalizeMultiaddrForCompare(addr)).toBe(addr); + }); +}); + +/** + * A configured bootstrap address is the user's own claim and its status row is how they + * debug it, so it must be probed for real rather than satisfied by any connection that + * happens to exist to the same peer. Gossiped addresses must not force: a peer naming + * many of them could otherwise make us open a connection per address. + */ +describe('addBootstrapPeers — forced probe only for configured addresses', () => { + function bareNetwork() { + const forced: boolean[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [{}], // already connected to this peer some other way + async dial(ma: { toString(): string }, opts?: { force?: boolean }): Promise { + forced.push(opts?.force === true); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, forced }; + } + + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + it('forces the dial for a configured address', async () => { + const { network, forced } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(forced).toEqual([true]); + }); + + it('does not force the dial for a discovered address', async () => { + const { network, forced } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(forced).toEqual([false]); + }); +}); + +/** + * "Is this configured infrastructure?" and "may we auto-evict it?" are the same + * question about the same fact. They used to be answered from two separate sets and + * only one of them was ever pruned, so a peer the user had already deleted from the + * bootstrap config kept its eviction exemption until the process restarted. + */ +describe('configured exemption ends when the peer leaves the config', () => { + function bareNetwork() { + const purged: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).noReachableSince = new Map([[PEER_ID, Date.now() - 45 * 60_000]]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]; + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(`/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`)]); + (network as any).addressProbeBackoff = new Map(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { getConnections: () => [] }; + (network as any).hasConnectionOtherThan = () => true; + (network as any).purgeStalePeer = async (pid: string): Promise => { + purged.push(pid); + }; + return { network, purged }; + } + + const undialable = { id: peerIdLike(PEER_ID), addresses: NO_ADDRESSES }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [undialable], 1); + + it('protects the peer while it is still configured', async () => { + const { network, purged } = bareNetwork(); + await run(network); + expect(purged).toEqual([]); + }); + + it('stops protecting it once the config entry is gone', async () => { + const { network, purged } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + await run(network); + expect(purged).toEqual([PEER_ID]); + }); + + it('drops the infrastructure status in the same step', async () => { + // The two used to be able to disagree; pruning must settle both at once. + const { network } = bareNetwork(); + expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(true); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect(network.isBootstrapOrRelayPeer(PEER_ID)).toBe(false); + }); + /** + * The autodial list is what zero-connection recovery walks. A bootstrap the user + * has deleted must leave it too, or the node keeps dialing that address every time + * it runs out of connections — the churn this work is supposed to end. + */ + it('forgets the deleted bootstrap address, so recovery stops dialing it', () => { + const { network } = bareNetwork(); + expect((network as any).bootstrapMultiaddrs).toHaveLength(1); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + }); + + it('also forgets it in the dedup set, so a later re-add can restore the address', () => { + const { network } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); + + it('leaves the addresses of other peers alone', () => { + const other = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const { network } = bareNetwork(); + (network as any).bootstrapMultiaddrs.push(multiaddr(`/ip4/203.0.113.10/tcp/9090/p2p/${other}`)); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([`/ip4/203.0.113.10/tcp/9090/p2p/${other}`]); + }); +}); + +/** + * The bootstrap list is walked one peer at a time and a single dial can take seconds. + * If the user edits that list — or leaves the network — mid-walk, the job started for + * the OLD list must stop: carrying on would re-add entries that are no longer + * configured AND re-mark them configured, which exempts them from the stale sweep for + * the rest of the process's life. That is precisely the resurrection this work exists + * to prevent, so the guard is checked here against a real second list entry. + */ +describe('addBootstrapPeers — superseded bootstrap configuration', () => { + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function bareNetwork(onFirstDial?: (network: Network) => void) { + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + // Model the edit landing while the FIRST dial is still in flight. + if (dialled.length === 1) onFirstDial?.(network); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { + async merge(): Promise {}, + }, + }; + return { network, dialled }; + } + + it('abandons the rest of the list when the network configuration is superseded', async () => { + const { network, dialled } = bareNetwork(n => n.bumpBootstrapGeneration('net-a')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A]); + }); + + it('does not re-mark the abandoned entry as configured', async () => { + const { network } = bareNetwork(n => n.bumpBootstrapGeneration('net-a')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_B)).toBe(false); + }); + + it('walks the whole list when nothing supersedes it', async () => { + const { network, dialled } = bareNetwork(); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A, ADDR_B]); + }); + + it('is not disturbed by an edit to a DIFFERENT network', async () => { + const { network, dialled } = bareNetwork(n => n.bumpBootstrapGeneration('net-other')); + await (network as any).addBootstrapPeers([ADDR_A, ADDR_B], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR_A, ADDR_B]); + }); +}); + +/** + * A dial already handed to libp2p cannot be recalled: hangUp closes connections that + * already exist, so a leave-network landing mid-dial finds nothing to close and the + * connection appears a moment after the cleanup finished. Abandoning the loop is not + * enough — the connection has to be closed too. + */ +describe('addBootstrapPeers — a dial that lands after leave-network', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(suppressed: string[]) { + const disconnected: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map([['net-a', new Set(suppressed)]]); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; + (network as any).disconnectPeer = async (peerID: string): Promise => { + disconnected.push(peerID); + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + // The leave happens while this dial is in flight. + (network as any).redialSuppressedByNet.get('net-a').add(PEER_ID); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, disconnected }; + } + + it('closes a connection that arrived after the peer was left', async () => { + const { network, disconnected } = bareNetwork([]); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([PEER_ID]); + }); + + /** + * Tearing the peer down is destructive — disconnectPeer suppresses re-dials and + * drops the peerStore entry — so leaving ONE network must not do it to a peer + * another joined network still has a claim on. Here the peer is configured + * infrastructure, which is claim enough. + */ + it('keeps a connection to a peer another joined network still needs', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + + it('keeps a connection to a peer that still subscribes another joined topic', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).pubsub = { getTopics: () => ['lish/net-b'], getSubscribers: () => [{ toString: () => PEER_ID }] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + + /** + * The peer we were dialing may never have been seen as a member of the network, so + * leave-network had nothing to hang up and never put it in the suppression set. + * Leaving the topic is the fact that decides it, not the suppression bookkeeping. + */ + it('closes a connection to a peer we never saw as a member of the network we left', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + (network as any).pubsub = { getTopics: () => [] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([PEER_ID]); + }); + + it('keeps the connection while the network is still joined', async () => { + const { network, disconnected } = bareNetwork([]); + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + (network as any).pubsub = { getTopics: () => ['lish/net-a'] }; + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(disconnected).toEqual([]); + }); + + it('leaves an ordinary dial connected', async () => { + const { network, disconnected } = bareNetwork([]); + // No leave lands this time: the dial does not add the peer to the suppression set. + (network as any).node.dial = async (ma: { toString(): string }): Promise => ({ remoteAddr: { toString: () => ma.toString() } }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(disconnected).toEqual([]); + }); +}); + +/** + * The autodial list is what zero-connection recovery dials. A gossip mention is only + * a claim, so an address that never answered must not end up on it — otherwise a peer + * naming many unreachable addresses parks them all there permanently, since an + * ordinary timeout has no cleanup path (only identity-mismatch does). + */ +describe('addBootstrapPeers — only a working discovered address joins the autodial list', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { fail?: boolean; remoteAddr?: string }) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (opts.fail) throw new Error('dial timeout'); + return { remoteAddr: { toString: () => opts.remoteAddr ?? ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('keeps a failed discovered address off the list', async () => { + const network = bareNetwork({ fail: true }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([]); + }); + + it('adds a discovered address that answered on the endpoint it claimed', async () => { + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([ADDR]); + }); + + it('keeps a discovered address off the list when the connection came back on another one', async () => { + const network = bareNetwork({ remoteAddr: `/ip4/198.51.100.1/tcp/4001/p2p/${PEER_ID}` }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(addresses(network)).toEqual([]); + }); + + /** + * A configured entry is user data: recovery has to keep trying it precisely while + * it is down, so it joins the list before the dial and stays even when that fails. + */ + it('keeps a failed configured address on the list', async () => { + const network = bareNetwork({ fail: true }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR]); + }); + + it('does not add the same address twice', async () => { + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR]); + }); + + /** + * The dedup used to be keyed on the peer id, so a bootstrap whose host the user + * edited was treated as already known and its new address never reached the list. + */ + it('adds a new address of a peer it already knows', async () => { + const moved = `/ip4/203.0.113.99/tcp/9090/p2p/${PEER_ID}`; + const network = bareNetwork({}); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + await (network as any).addBootstrapPeers([moved], 'net-a', 'configured'); + expect(addresses(network)).toEqual([ADDR, moved]); + }); +}); + +/** + * Whether an address is dialable is a fact about THIS HOST right now — a LAN or VPN + * bootstrap stops passing the routability filter the moment that interface drops. + * Whether the user configured a peer is a fact about the saved config. Deriving the + * second from the first left a VPN bootstrap unregistered whenever the tunnel was + * down at startup, and an unregistered configured peer loses the exemption that is + * supposed to make it un-evictable. + */ +describe('addBootstrapPeers — a non-routable configured entry is still configured', () => { + // A private address in a subnet this host is not on: the filter rejects it. + const OFF_VPN = `/ip4/10.201.0.5/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const outcomes: Array<{ status: string; message: string | null }> = []; + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome(_n: unknown, _a: unknown, _p: unknown, status: string, message: string | null) { + outcomes.push({ status, message }); + }, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, outcomes, dialled }; + } + + it('registers the peer as configured even though the address is not routable', async () => { + const { network } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + it('still does not dial an address the filter rejected', async () => { + const { network, dialled } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect(dialled).toEqual([]); + }); + + it('tells the user why the configured entry is doing nothing', async () => { + const { network, outcomes } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect(outcomes).toEqual([{ status: 'error', message: 'address is not routable from this host' }]); + }); + + /** + * It has to be on the recovery list even while unroutable, or nothing retries it + * when the interface comes back. Recovery re-checks routability before dialing. + */ + it('still puts the unroutable configured address on the recovery list', async () => { + const { network } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'configured'); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([OFF_VPN]); + }); + + it('says nothing about a discovered address the filter rejected', async () => { + const { network, outcomes } = bareNetwork(); + await (network as any).addBootstrapPeers([OFF_VPN], 'net-a', 'discovered'); + expect(outcomes).toEqual([]); + expect((network as any).configuredBootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); +}); + +/** + * An expired quarantine buys exactly one probe. Letting a failed probe pass without + * closing the window again means every later gossip mention spends another dial and + * refreshes the status row — the churn the quarantine exists to stop. + */ +describe('addBootstrapPeers — quarantine after the probe it allowed', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(fail: boolean, quarantinedAt: number) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map([[PEER_ID, quarantinedAt]]); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (fail) throw new Error('dial timeout'); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const LONG_AGO = Date.now() - 10 * 60 * 60_000; + + it('re-arms the quarantine when the allowed probe fails', async () => { + const network = bareNetwork(true, LONG_AGO); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + const at = (network as any).unreachableQuarantine.get(PEER_ID); + expect(at).toBeGreaterThan(LONG_AGO); + }); + + it('leaves the quarantine lifted when the probe succeeds', async () => { + const network = bareNetwork(false, LONG_AGO); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); + + it('does not quarantine a plain failure that was never in one', async () => { + const network = bareNetwork(true, LONG_AGO); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); +}); + +/** + * Zero-connection recovery used to dial every address it held, ignoring the pacing + * re-dial maintenance had just applied. On an isolated node that meant a dead + * discovered peer was re-dialed every tick forever, because maintenance stops + * counting failures the moment there is no other connection to prove we are online. + */ +describe('isRecoveryDialDue', () => { + const now = 1_000_000; + const none = new Map(); + const noBackoff = new Map(); + + it('allows a peer with no backoff and no quarantine', () => { + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, none)).toBe(true); + }); + + it('holds a peer back inside its backoff window and releases it after', () => { + expect(isRecoveryDialDue(PEER_ID, now, new Map([[PEER_ID, { nextAttempt: now + 1 }]]), none)).toBe(false); + expect(isRecoveryDialDue(PEER_ID, now, new Map([[PEER_ID, { nextAttempt: now }]]), none)).toBe(true); + }); + + it('holds a quarantined peer back until the window passes', () => { + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, new Map([[PEER_ID, now - 60_000]]))).toBe(false); + expect(isRecoveryDialDue(PEER_ID, now, noBackoff, new Map([[PEER_ID, now - 10 * 60 * 60_000]]))).toBe(true); + }); +}); + +/** + * Noise proves one thing only: THIS address no longer leads to the peer we expected. + * Purging the whole peer because it happened to be disconnected threw away addresses + * that were never disproved — a peer reachable tomorrow on its other address was + * forgotten because one stale entry answered with the wrong identity today. + */ +describe('addBootstrapPeers — identity mismatch trims the address, not the peer', () => { + const BAD = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const GOOD = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(storedAddresses: string[]) { + const purged: string[] = []; + const patched: string[][] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = [multiaddr(BAD)]; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; + (network as any).purgeStalePeer = async (id: string): Promise => { + purged.push(id); + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(): Promise { + throw new Error(`Payload identity key 12D3KooWSomeoneElseSomeoneElseSomeoneElseSomeoneEls does not match expected remote identity key ${PEER_ID}`); + }, + peerStore: { + async get(): Promise { + return { addresses: storedAddresses.map(a => ({ multiaddr: multiaddr(a) })) }; + }, + async patch(_pid: unknown, data: { multiaddrs: Array<{ toString(): string }> }): Promise { + patched.push(data.multiaddrs.map(m => m.toString())); + }, + async merge(): Promise {}, + }, + }; + return { network, purged, patched }; + } + + it('keeps a disconnected peer that still has an undisproved address', async () => { + const { network, purged, patched } = bareNetwork([BAD, GOOD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect(purged).toEqual([]); + expect(patched).toEqual([[GOOD]]); + }); + + it('purges only once nothing usable is left', async () => { + const { network, purged } = bareNetwork([BAD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect(purged).toEqual([PEER_ID]); + }); + + it('drops the disproved address from the autodial list either way', async () => { + const { network } = bareNetwork([BAD, GOOD]); + await (network as any).addBootstrapPeers([BAD], 'net-a', 'configured'); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + }); +}); + +/** + * An address the routability filter rejected never reaches the peerStore, so re-dial + * maintenance has no candidate for it, and zero-connection recovery only runs with NO + * connections at all. Without this slow pass a node talking to someone else would + * never notice a VPN bootstrap became reachable again. + */ +describe('probeParkedConfiguredBootstraps', () => { + const PARKED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + /** A second configured address of the SAME peer — the sibling finding 4 is about. */ + const SIBLING = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { configured?: boolean; addresses?: string[]; connectionAddrs?: string[]; failAddresses?: string[] } = {}) { + const dialed: string[] = []; + const addresses = opts.addresses ?? [PARKED]; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured === false ? [] : [PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set(opts.configured === false ? [] : addresses.map(a => normalizeMultiaddrForCompare(a))); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).bootstrapMultiaddrs = addresses.map(a => multiaddr(a)); + const repaired: string[] = []; + (network as any).bootstrapTracker = { + recordAddressReachable(address: string) { + repaired.push(address); + }, + }; + (network as any).node = { + getConnections: () => (opts.connectionAddrs ?? []).map(a => ({ remoteAddr: { toString: () => a } })), + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + if (opts.failAddresses?.includes(ma.toString())) throw new Error('dial timeout'); + }, + }; + return { network, dialed, repaired }; + } + + const run = (network: Network): Promise => (network as any).probeParkedConfiguredBootstraps(1); + + it('probes a configured address even though we hold other connections', async () => { + const { network, dialed } = bareNetwork({ connectionAddrs: [`/ip4/198.51.100.200/tcp/9090/p2p/${PEER_ID}`] }); + await run(network); + expect(dialed).toEqual([multiaddr(PARKED).toString()]); + }); + + /** + * This probe is the only thing that ever retries an address the routability filter + * rejected at configure time — a LAN or VPN bootstrap whose interface was down. The + * `error` row written back then had no other way to go green again. + */ + it('repairs the status row when a parked address answers', async () => { + const { network, repaired } = bareNetwork(); + await run(network); + expect(repaired).toEqual([multiaddr(PARKED).toString()]); + }); + + it('leaves the row alone while the address is still failing', async () => { + const { network, repaired } = bareNetwork({ failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + expect(repaired).toEqual([]); + }); + + it('leaves a discovered address to the loops that own it', async () => { + const { network, dialed } = bareNetwork({ configured: false }); + await run(network); + expect(dialed).toEqual([]); + }); + + /** + * Only a connection ON THIS ENDPOINT answers what the probe asks. A connection to the + * same peer over another address used to skip it — which is how a broken configured + * entry kept looking fine for as long as the peer was reachable some other way. + */ + it('skips only when the existing connection is on this very address', async () => { + const { network, dialed } = bareNetwork({ connectionAddrs: [PARKED] }); + await run(network); + expect(dialed).toEqual([]); + }); + + it('respects the backoff so a broken entry costs one dial per window', async () => { + const { network, dialed } = bareNetwork(); + (network as any).addressProbeBackoff = new Map([[normalizeMultiaddrForCompare(PARKED), { nextAttempt: Date.now() + 60_000, failCount: 1 }]]); + await run(network); + expect(dialed).toEqual([]); + }); + + it('skips a peer the user left', async () => { + const { network, dialed } = bareNetwork(); + (network as any).redialSuppressedByNet = new Map([['net-a', new Set([PEER_ID])]]); + await run(network); + expect(dialed).toEqual([]); + }); + + /** + * The backoff used to be keyed by PEER while the loop iterates by ADDRESS: the dead + * address failed, the whole peer went into backoff, its working sibling was skipped + * for the rest of the pass — and the next pass started at the dead one again, so the + * sibling could go untried indefinitely. + */ + it('tries the sibling address of a peer whose other address just failed', async () => { + const { network, dialed } = bareNetwork({ addresses: [PARKED, SIBLING], failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + expect(dialed).toEqual([multiaddr(PARKED).toString(), multiaddr(SIBLING).toString()]); + }); + + it('paces each address on its own record', async () => { + const { network } = bareNetwork({ addresses: [PARKED, SIBLING], failAddresses: [multiaddr(PARKED).toString()] }); + await run(network); + const backoff = (network as any).addressProbeBackoff as Map; + expect(backoff.has(normalizeMultiaddrForCompare(PARKED))).toBe(true); + expect(backoff.has(normalizeMultiaddrForCompare(SIBLING))).toBe(false); + }); + + /** + * Without `force` libp2p hands back whatever connection it already holds to the peer, + * so the probe would resolve without ever touching the address it is asking about. + */ + it('forces the dial so the address itself is contacted', async () => { + const forced: boolean[] = []; + const { network } = bareNetwork(); + (network as any).node.dial = async (_ma: unknown, opts?: { force?: boolean }): Promise => { + forced.push(opts?.force === true); + }; + await run(network); + expect(forced).toEqual([true]); + }); +}); + +/** + * One peer can hold a configured address AND a gossip-learned one at the same time. + * Deciding "is this configured" from the peer id let the gossip-learned sibling + * inherit the configured exemption — no backoff, no quarantine, and it survived the + * user deleting the configured entry it had nothing to do with. + */ +describe('configured origin is a property of the address, not the peer', () => { + const CONFIGURED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const DISCOVERED = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(CONFIGURED)]); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED), multiaddr(DISCOVERED)]; + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; + (network as any).node = { + getPeers: () => [], + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + }, + }; + return { network, dialed }; + } + + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('paces a discovered address whose peer is configured under a DIFFERENT address', async () => { + // Only the discovered address is on the list, so nothing else can satisfy the + // loop: if it gets dialed while inside its backoff window, it was wrongly given + // the configured exemption its sibling address owns. + const { network, dialed } = bareNetwork(); + (network as any).bootstrapMultiaddrs = [multiaddr(DISCOVERED)]; + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + await (network as any).runZeroConnectionRecovery(); + expect(dialed).toEqual([]); + }); + + it('still lets the configured address through the same backoff', async () => { + const { network, dialed } = bareNetwork(); + (network as any).bootstrapMultiaddrs = [multiaddr(CONFIGURED)]; + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() + 60_000, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }]]); + await (network as any).runZeroConnectionRecovery(); + expect(dialed).toEqual([multiaddr(CONFIGURED).toString()]); + }); + + it('keeps the discovered address when the configured one is deleted', () => { + const { network } = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect(addresses(network)).toEqual([multiaddr(DISCOVERED).toString()]); + }); +}); + +/** + * stop() clears the per-run state so a restart starts clean. The slow-cadence counter + * was left out, so a fresh node could inherit a count that made its very first tick the + * slow one — the opposite of the ownership the epoch guards enforce everywhere else. + * The delayed peer-count probes were likewise untracked and kept firing at a node the + * run no longer owned. + */ +describe('Network.stop — per-run state really is per run', () => { + function bareNetwork() { + const network = Object.create(Network.prototype) as Network; + for (const field of ['lastWantResponseTime', 'seenSearchIDs', 'topicHandlers', 'dcutrPeers', 'bootstrapPeerIDs', 'bootstrapGeneration', '_lastPeerCounts', '_lastScores', 'redialBackoff', 'unreachableQuarantine', 'addressProbeBackoff', 'noReachableSince', 'configuredBootstrapPeerIDs', 'configuredBootstrapAddresses', 'redialSuppressedByNet', 'pxIngressLogKeys', 'inFlightBootstrapDials']) { + (network as any)[field] = field === 'seenSearchIDs' || field === 'dcutrPeers' || field === 'bootstrapPeerIDs' || field === 'configuredBootstrapPeerIDs' || field === 'configuredBootstrapAddresses' || field === 'inFlightBootstrapDials' ? new Set() : new Map(); + } + (network as any).runEpoch = 1; + (network as any).statusInterval = null; + (network as any).wantResponseCleanupInterval = null; + (network as any)._peerCountDebounceTimer = null; + (network as any).listeners = []; + (network as any).bootstrapMultiaddrs = []; + (network as any).delayedPeerCountTimers = new Set(); + (network as any).peerAnnounce = { stop() {} }; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + clear() {}, + }; + (network as any).node = null; + (network as any).datastore = null; + // stop() runs under the lifecycle mutex; a prototype-only instance has no field + // initializers, so it has to be supplied here. + (network as any).lifecycle = 'running'; + (network as any).lifecycleMutex = new Mutex(); + return network; + } + + it('resets the slow-cadence tick counter', async () => { + const network = bareNetwork(); + (network as any).statusTickCount = 4; // one short of the slow tick + await network.stop(); + expect((network as any).statusTickCount).toBe(0); + }); + + /** + * The epoch guard already stops a late probe from DOING anything, so observing the + * callback proves nothing about cancellation. What has to be asserted is that the + * handle is actually released — otherwise a pending timer keeps a closure on the old + * instance alive until it fires. + */ + it('cancels the delayed peer-count probes it armed', async () => { + const network = bareNetwork(); + (network as any).armDelayedPeerCountCheck(60_000); + (network as any).armDelayedPeerCountCheck(60_000); + const armed = [...(network as any).delayedPeerCountTimers]; + expect(armed).toHaveLength(2); + + const realClearTimeout = globalThis.clearTimeout; + const cleared: unknown[] = []; + globalThis.clearTimeout = ((timer: unknown) => { + cleared.push(timer); + return realClearTimeout(timer as Parameters[0]); + }) as typeof globalThis.clearTimeout; + try { + await network.stop(); + } finally { + globalThis.clearTimeout = realClearTimeout; + } + + for (const timer of armed) expect(cleared).toContain(timer); + expect((network as any).delayedPeerCountTimers.size).toBe(0); + }); +}); + +/** + * Zero-connection recovery used to work off the peer list the status tick snapshotted at + * its start — BEFORE re-dial maintenance ran. Maintenance reconnecting a peer in the + * meantime left recovery still believing it was isolated, so it dialed anyway. + */ +describe('runZeroConnectionRecovery — connectivity is read, not remembered', () => { + const ADDR_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const PEER_B = '12D3KooWPvH1oQjQZS8TtucG4NsW2PsnW87jwMAiRLKgrNGS17fp'; + const ADDR_B = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_B}`; + + function bareNetwork(opts: { addresses?: string[]; peers?: () => unknown[]; onDial?: (address: string) => void } = {}) { + const dialed: string[] = []; + // The churn dump is the first thing the loop does, so it witnesses whether the + // isolation check at the top of the function ran at all — a later check inside the + // loop would already have let the misleading "No connections" report out. + let churnDumps = 0; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapMultiaddrs = (opts.addresses ?? [ADDR_A]).map(a => multiaddr(a)); + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => { + churnDumps++; + return []; + }, + }; + (network as any).node = { + getPeers: opts.peers ?? ((): unknown[] => []), + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + opts.onDial?.(ma.toString()); + }, + }; + return { network, dialed, churn: () => churnDumps }; + } + + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(1); + + it('does not dial at all when a peer connected since the tick began', async () => { + const { network, dialed, churn } = bareNetwork({ peers: () => [{ toString: () => PEER_B }] }); + await run(network); + expect(dialed).toEqual([]); + expect(churn()).toBe(0); + }); + + it('stops the pass as soon as a connection exists', async () => { + // The first dial fails, but an inbound connection lands during it; the second + // address must not be tried, because the node is no longer isolated. + let connected = false; + const { network, dialed } = bareNetwork({ + addresses: [ADDR_A, ADDR_B], + peers: (): unknown[] => (connected ? [{ toString: () => PEER_B }] : []), + onDial: () => { + connected = true; + throw new Error('dial timeout'); + }, + }); + await run(network); + expect(dialed).toEqual([multiaddr(ADDR_A).toString()]); + }); +}); + +/** + * The recovery loop used to only LOG its failures. For an address whose peer is not in + * the peerStore, re-dial maintenance never sees the peer either — so nothing anywhere + * paced it and an isolated node re-dialed a dead entry on every 30 s tick, forever. + */ +describe('runZeroConnectionRecovery — a failed dial paces the next one', () => { + const DISCOVERED = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const CONFIGURED = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(opts: { address: string; configured: boolean }) { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).addressProbeBackoff = new Map(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(opts.configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(opts.configured ? [normalizeMultiaddrForCompare(opts.address)] : []); + (network as any).bootstrapMultiaddrs = [multiaddr(opts.address)]; + (network as any).recentDisconnects = []; + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + entries: () => [], + }; + (network as any).node = { + getPeers: (): unknown[] => [], + async dial(ma: { toString(): string }): Promise { + dialed.push(ma.toString()); + throw new Error('dial timeout'); + }, + }; + return { network, dialed }; + } + + const run = (network: Network): Promise => (network as any).runZeroConnectionRecovery(1); + + it('writes the full four-field backoff record for a discovered address', async () => { + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + await run(network); + const entry = (network as any).redialBackoff.get(PEER_ID) as { nextAttempt: number; failCount: number; firstFailure: number; evictionFails: number } | undefined; + expect(entry).toBeDefined(); + expect(Object.keys(entry!).sort()).toEqual(['evictionFails', 'failCount', 'firstFailure', 'nextAttempt']); + expect(entry!.failCount).toBe(1); + expect(entry!.nextAttempt).toBeGreaterThan(Date.now()); + }); + + it('skips the same address on the very next pass', async () => { + const { network, dialed } = bareNetwork({ address: DISCOVERED, configured: false }); + await run(network); + await run(network); + expect(dialed).toEqual([multiaddr(DISCOVERED).toString()]); + }); + + /** + * At zero connections we cannot tell the remote apart from our own outage, which is + * the exact condition nextEvictionFailCount resets on — so a recovery failure must + * never become evidence against the peer. + */ + it('does not count the failure towards eviction', async () => { + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + (network as any).redialBackoff = new Map([[PEER_ID, { nextAttempt: Date.now() - 1, failCount: 2, firstFailure: Date.now() - 60_000, evictionFails: 3 }]]); + await run(network); + expect(((network as any).redialBackoff.get(PEER_ID) as { evictionFails: number }).evictionFails).toBe(3); + }); + + it('re-arms an expired quarantine that let the dial through', async () => { + const longAgo = Date.now() - 10 * 60 * 60_000; + const { network } = bareNetwork({ address: DISCOVERED, configured: false }); + (network as any).unreachableQuarantine = new Map([[PEER_ID, longAgo]]); + await run(network); + expect((network as any).unreachableQuarantine.get(PEER_ID)).toBeGreaterThan(longAgo); + }); + + /** + * Configured entries stay exempt from eviction and from quarantine — but exempt is + * not unlimited: several dead ones at a 10 s timeout each turn every tick into + * minutes of back-to-back dialing. + */ + it('paces a configured address too, on its own record', async () => { + const { network, dialed } = bareNetwork({ address: CONFIGURED, configured: true }); + await run(network); + await run(network); + expect(dialed).toEqual([multiaddr(CONFIGURED).toString()]); + }); + + it('keeps the configured wait well under the general re-dial ceiling', async () => { + const { network } = bareNetwork({ address: CONFIGURED, configured: true }); + const key = normalizeMultiaddrForCompare(CONFIGURED); + for (let failCount = 0; failCount < 12; failCount++) { + (network as any).addressProbeBackoff.set(key, { nextAttempt: 0, failCount }); + await run(network); + } + const entry = (network as any).addressProbeBackoff.get(key) as { nextAttempt: number }; + expect(entry.nextAttempt - Date.now()).toBeLessThanOrEqual(5 * 60_000); + }); + + it('leaves the per-peer eviction record untouched for a configured address', async () => { + const { network } = bareNetwork({ address: CONFIGURED, configured: true }); + await run(network); + expect((network as any).redialBackoff.size).toBe(0); + }); +}); + +/** + * An evicted peer is normally gone from the peerStore, so it cannot become a re-dial + * candidate at all — but that delete is best-effort and mDNS, identify and peer-announce + * can all put the entry straight back. Without a quarantine check here, the peer we just + * wrote off is dialed again on the very next tick. + */ +describe('runRedialMaintenance — quarantined peers are not candidates', () => { + function bareNetwork(quarantinedAt: number | null, configured = false) { + const dialed: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialBackoff = new Map(); + (network as any).redialSuppressedByNet = new Map(); + (network as any).unreachableQuarantine = quarantinedAt === null ? new Map() : new Map([[PEER_ID, quarantinedAt]]); + (network as any).noReachableSince = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(configured ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + deleteDiscoveredByPeerID() {}, + }; + (network as any).pubsub = { getTopics: () => [], getSubscribers: () => [] }; + (network as any).node = { + getConnections: () => [], + async dial(id: { toString(): string }): Promise { + dialed.push(id.toString()); + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, dialed }; + } + + // A documentation-range public address, so the dial gater lets it through wherever + // this test happens to run. + const peer = { id: peerIdLike(PEER_ID), addresses: [{ multiaddr: multiaddr('/ip4/203.0.113.5/tcp/9090') }] }; + const run = (network: Network): Promise => (network as any).runRedialMaintenance([], [peer], 1); + + it('skips a peer still inside its unreachable quarantine', async () => { + const { network, dialed } = bareNetwork(Date.now() - 60_000); + await run(network); + expect(dialed).toEqual([]); + }); + + it('dials it again once the quarantine window has passed', async () => { + const { network, dialed } = bareNetwork(Date.now() - 10 * 60 * 60_000); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); + + it('never holds a configured peer back', async () => { + const { network, dialed } = bareNetwork(Date.now() - 60_000, true); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); + + it('dials a peer that was never quarantined', async () => { + const { network, dialed } = bareNetwork(null); + await run(network); + expect(dialed).toEqual([PEER_ID]); + }); +}); + +/** + * The status tracker keeps the STRONGER origin when a row is overwritten, so a gossip + * re-announcement of an address the user configured lands on a configured row. The dial + * followed the caller's origin instead: it went unforced, libp2p handed back the + * connection the peer held on a DIFFERENT address, and the discovered branch recorded + * 'connected' — a green light on a configured address that was never contacted. + */ +describe('addBootstrapPeers — a gossip announce of a configured address', () => { + const CONFIGURED_A = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const WORKING_B = `/ip4/198.51.100.7/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(knownConfigured: string[]) { + const outcomes: string[] = []; + const forced: boolean[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(knownConfigured.length > 0 ? [PEER_ID] : []); + (network as any).configuredBootstrapAddresses = new Set(knownConfigured.map(a => normalizeMultiaddrForCompare(a))); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome(_net: unknown, _addr: unknown, _pid: unknown, status: string) { + outcomes.push(status); + }, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [{}], // the peer was reachable via WORKING_B all along + async dial(_ma: unknown, opts?: { force?: boolean }): Promise { + forced.push(opts?.force === true); + // libp2p answers with the connection its OTHER address won. + return { remoteAddr: { toString: () => WORKING_B } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, outcomes, forced }; + } + + it('does not turn the configured row green off an unverified dial', async () => { + const { network, outcomes } = bareNetwork([CONFIGURED_A]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(outcomes).toEqual([]); + }); + + it('probes the address for real, as the configured branch would', async () => { + const { network, forced } = bareNetwork([CONFIGURED_A]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(forced).toEqual([true]); + }); + + it('still treats a genuinely unknown address as discovered', async () => { + const { network, outcomes, forced } = bareNetwork([]); + await (network as any).addBootstrapPeers([CONFIGURED_A], 'net-a', 'discovered'); + expect(forced).toEqual([false]); + expect(outcomes).toEqual(['connected']); + }); +}); + +/** + * purgeStalePeer takes four things away — the bootstrap dedup entry, the peer's + * addresses on the autodial list, its gossipsub direct entry and its keep-alive tag. + * The TOCTOU healing branch put only some of them back, and periodic promotion then + * skipped the peer precisely BECAUSE it was in bootstrapPeerIDs again, so the missing + * pieces were never filled in. + */ +describe('purgeStalePeer — healing an inbound race restores the whole dial state', () => { + const REMOTE = '/ip4/203.0.113.9/tcp/9090'; + + function bareNetwork(suppressed: string[] = []) { + const flagDuringDirectAdd: boolean[] = []; + const merges: Array> = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).redialBackoff = new Map(); + (network as any).unreachableQuarantine = new Map([[PEER_ID, Date.now()]]); + (network as any).redialSuppressedByNet = new Map(suppressed.length > 0 ? [['net-a', new Set(suppressed)]] : []); + const direct = new Set(); + (network as any).pubsub = { + direct: { + add(id: string) { + // Ordering probe: bootstrapPeerIDs is the flag other paths read as + // "this peer is handled", so it must still be unset here. + flagDuringDirectAdd.push((network as any).bootstrapPeerIDs.has(PEER_ID)); + direct.add(id); + }, + delete: (id: string): boolean => direct.delete(id), + has: (id: string): boolean => direct.has(id), + }, + }; + (network as any).node = { + // The inbound connection that raced the purge is present throughout. + getConnections: () => [{ remoteAddr: multiaddr(REMOTE), async close(): Promise {} }], + peerStore: { + async delete(): Promise {}, + async merge(_pid: unknown, patch: Record): Promise { + merges.push(patch); + }, + }, + }; + return { network, direct, merges, flagDuringDirectAdd }; + } + + const run = (network: Network): Promise => (network as any).purgeStalePeer(PEER_ID, 'test', 1); + + it('puts the peer back in the bootstrap dedup set', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + it('puts its address back on the autodial list, carrying the peer identity', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString())).toEqual([`${REMOTE}/p2p/${PEER_ID}`]); + }); + + it('puts it back in the gossipsub fast-reconnect set', async () => { + const { network, direct } = bareNetwork(); + await run(network); + expect(direct.has(PEER_ID)).toBe(true); + }); + + it('re-stamps the keep-alive tag', async () => { + const { network, merges } = bareNetwork(); + await run(network); + expect(merges).toHaveLength(1); + expect(merges[0]).toHaveProperty('tags'); + }); + + it('sets the bootstrap dedup flag last, so nothing can observe a half-healed peer', async () => { + const { network, flagDuringDirectAdd } = bareNetwork(); + await run(network); + expect(flagDuringDirectAdd).toEqual([false]); + }); + + it('lifts the unreachable quarantine', async () => { + const { network } = bareNetwork(); + await run(network); + expect((network as any).unreachableQuarantine.has(PEER_ID)).toBe(false); + }); + + /** + * A peer hung up by leave-network is meant to be forgotten. A connection racing the + * purge is not a reason to rebuild the dial state the leave deliberately tore down. + */ + it('does not heal a peer the user left', async () => { + const { network, direct } = bareNetwork([PEER_ID]); + await run(network); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + expect((network as any).bootstrapMultiaddrs).toEqual([]); + expect(direct.has(PEER_ID)).toBe(false); + }); +}); + +/** + * Gossip re-announces a dead peer on every cycle. The intake path used to answer each + * mention with a fresh dial because it consulted the unreachable quarantine and nothing + * else — the per-peer backoff that paces every other dial path was never read, and never + * written either, so it could not have bitten even if it had been. + */ +describe('addBootstrapPeers — discovered dials are paced by the per-peer backoff', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(dialOutcome: 'ok' | 'fail') { + const dialled: string[] = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + if (dialOutcome === 'fail') throw new Error('dial timed out'); + return { remoteAddr: { toString: () => ADDR } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { network, dialled }; + } + + it('records a failed discovered dial into the shared backoff', async () => { + const { network } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).redialBackoff.get(PEER_ID)?.nextAttempt).toBeGreaterThan(Date.now()); + }); + + it('refuses a second mention of the same peer while the backoff is running', async () => { + const { network, dialled } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(dialled).toEqual([ADDR]); + }); + + it('dials again once the backoff window has passed', async () => { + const { network, dialled } = bareNetwork('fail'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() - 1, failCount: 1, firstFailure: Date.now(), evictionFails: 0 }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect(dialled).toEqual([ADDR, ADDR]); + }); + + /** A configured entry is user data and the way back in — the backoff must not hold it. */ + it('never holds a configured address back on the peer backoff', async () => { + const { network, dialled } = bareNetwork('fail'); + (network as any).redialBackoff.set(PEER_ID, { nextAttempt: Date.now() + 600_000, failCount: 9, firstFailure: Date.now(), evictionFails: 0 }); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect(dialled).toEqual([ADDR]); + }); + + /** A dial that worked clears the record, so a returning peer is not paced. */ + it('leaves no backoff behind after a successful discovered dial', async () => { + const { network } = bareNetwork('ok'); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).redialBackoff.has(PEER_ID)).toBe(false); + }); +}); + +/** + * The pubsub dispatcher does not await the announce handler, so two announces naming the + * same address run their intake concurrently. Each used to spend its own 10 s dial + * timeout on one endpoint, and the peer backoff cannot help — it is only written once a + * dial has already failed. + */ +describe('addBootstrapPeers — one dial per address at a time', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const OTHER_ADDR = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const dialled: string[] = []; + // Every dial parks here until released, so a second intake run genuinely overlaps + // the first instead of merely following it. + const pending: Array<() => void> = []; + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + dialled.push(ma.toString()); + await new Promise(resolve => pending.push(resolve)); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return { + network, + dialled, + releaseDials: (): void => { + for (const resolve of pending.splice(0)) resolve(); + }, + }; + } + + it('drops a second intake run while the first is still dialing the address', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); // let the first run reach its dial + // Not awaited: without the claim the second run parks on its own dial, and awaiting + // it here would hang the test instead of failing the assertion below. + const second = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + + expect(dialled).toEqual([ADDR]); + releaseDials(); + await Promise.all([first, second]); + }); + + it('still dials a different address of the same peer concurrently', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + const second = (network as any).addBootstrapPeers([OTHER_ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + + expect(dialled).toEqual([ADDR, OTHER_ADDR]); + releaseDials(); + await Promise.all([first, second]); + }); + + it('releases the claim once the dial settles, so a later mention can dial again', async () => { + const { network, dialled, releaseDials } = bareNetwork(); + const first = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + releaseDials(); + await first; + const second = (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + await Bun.sleep(1); + releaseDials(); + await second; + + expect(dialled).toEqual([ADDR, ADDR]); + }); +}); + +/** + * `bootstrapPeerIDs` is a global, unbounded, never-TTL'd set that other code reads as + * "this peer is handled" — and the libp2p config closure reads it too. Admitting an + * identity the moment gossip named it meant any topic subscriber could put arbitrary peer + * IDs into it, before anything had shown the identity even exists. + */ +describe('addBootstrapPeers — an identity joins the bootstrap set only once it answers', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(dialFails: boolean) { + const network = Object.create(Network.prototype) as Network; + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = { + batchDebounced(_net: string, fn: () => Promise): Promise { + return fn(); + }, + markPending() {}, + recordOutcome() {}, + deletePeer() {}, + }; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + if (dialFails) throw new Error('dial timed out'); + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + it('leaves an announced identity out while its dial is failing', async () => { + const network = bareNetwork(true); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(false); + }); + + it('admits the identity once the peer answers', async () => { + const network = bareNetwork(false); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'discovered'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); + + /** A configured identity is the user's own assertion and does not wait for a dial. */ + it('admits a configured identity even when its address is down', async () => { + const network = bareNetwork(true); + await (network as any).addBootstrapPeers([ADDR], 'net-a', 'configured'); + expect((network as any).bootstrapPeerIDs.has(PEER_ID)).toBe(true); + }); +}); + +/** + * The configured-address probe backoff is keyed by address, so it has to be released + * with the address. Left behind it grows across every configuration change, and a + * re-added address inherits the deleted entry's failCount and its multi-minute + * nextAttempt — the user deletes an entry, adds it back, and nothing dials it. + */ +describe('configured bootstrap removal releases the address probe backoff', () => { + const ADDR = `/ip4/203.0.113.9/tcp/9090/p2p/${PEER_ID}`; + const OTHER = `/ip4/203.0.113.10/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork() { + const network = Object.create(Network.prototype) as Network; + (network as any).configuredBootstrapPeerIDs = new Set([PEER_ID]); + (network as any).configuredBootstrapAddresses = new Set([normalizeMultiaddrForCompare(ADDR), normalizeMultiaddrForCompare(OTHER)]); + (network as any).bootstrapPeerIDs = new Set([PEER_ID]); + (network as any).bootstrapMultiaddrs = [multiaddr(ADDR), multiaddr(OTHER)]; + (network as any).addressProbeBackoff = new Map([ + [normalizeMultiaddrForCompare(ADDR), { nextAttempt: Date.now() + 300_000, failCount: 5 }], + [normalizeMultiaddrForCompare(OTHER), { nextAttempt: Date.now() + 300_000, failCount: 5 }], + ]); + return network; + } + + it('forgets the backoff of an address removed from the configuration', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(ADDR))).toBe(false); + }); + + it('keeps the backoff of an address that stayed', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(OTHER))).toBe(true); + }); + + it('forgets the backoff of every configured address of a removed peer', () => { + const network = bareNetwork(); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect([...(network as any).addressProbeBackoff.keys()]).toEqual([]); + }); + + /** A gossip-learned address of the same peer is not the user's to lose — nor its pacing. */ + it('leaves a discovered address of the removed peer alone', () => { + const network = bareNetwork(); + (network as any).configuredBootstrapAddresses.delete(normalizeMultiaddrForCompare(OTHER)); + network.pruneConfiguredBootstrapPeer(PEER_ID); + expect((network as any).addressProbeBackoff.has(normalizeMultiaddrForCompare(OTHER))).toBe(true); + }); + + /** The user deletes an entry and puts it straight back: it must be dialed at once. */ + it('lets a re-added address be probed immediately', () => { + const network = bareNetwork(); + network.pruneBootstrapAddresses([ADDR]); + expect((network as any).isAddressProbeDue(normalizeMultiaddrForCompare(ADDR), Date.now())).toBe(true); + }); +}); + +/** + * Intake writes two status rows per address — a pending mark and an outcome — and each + * used to rebuild and publish the network's whole peer list. A 128-address announce cost + * 256 snapshots and 256 pushes, all but the last thrown away by the UI. + */ +describe('addBootstrapPeers — status updates are grouped per run', () => { + function bareNetwork(emissions: number[]) { + const network = Object.create(Network.prototype) as Network; + const tracker = new BootstrapStatusTracker(); + tracker.setOnChange((_networkID, status) => emissions.push(status.peers.length)); + (network as any).runEpoch = 1; + (network as any).redialSuppressedByNet = new Map(); + (network as any).configuredBootstrapPeerIDs = new Set(); + (network as any).configuredBootstrapAddresses = new Set(); + (network as any).unreachableQuarantine = new Map(); + (network as any).redialBackoff = new Map(); + (network as any).bootstrapPeerIDs = new Set(); + (network as any).bootstrapMultiaddrs = []; + (network as any).bootstrapGeneration = new Map(); + (network as any).inFlightBootstrapDials = new Set(); + (network as any).bootstrapTracker = tracker; + (network as any).node = { + peerId: { toString: () => 'selfID' }, + getConnections: () => [], + async dial(ma: { toString(): string }): Promise { + return { remoteAddr: { toString: () => ma.toString() } }; + }, + peerStore: { async merge(): Promise {} }, + }; + return network; + } + + const addrs = (count: number): string[] => Array.from({ length: count }, (_v, i) => `/ip4/203.0.113.${i + 1}/tcp/9090/p2p/${PEER_ID}`); + + it('publishes one snapshot for a whole announce instead of two per address', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(20), 'net-a', 'discovered'); + + expect(emissions).toEqual([20]); + }); + + it('still records every address', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(20), 'net-a', 'discovered'); + + expect((network as any).bootstrapTracker.getStatus('net-a').peers).toHaveLength(20); + }); + + /** No owning network means no status rows at all — the wrapper must not assume one. */ + it('runs unbatched when there is no network to group under', async () => { + const emissions: number[] = []; + const network = bareNetwork(emissions); + + await (network as any).addBootstrapPeers(addrs(3), null, 'discovered'); + + expect(emissions).toEqual([]); + }); +}); + +/** + * The autodial list grows with every discovered endpoint that has ever answered, and + * nothing but an identity purge ever shortens it — so a network with churn of reachable + * one-off peers inflates it for the lifetime of the process, and zero-connection recovery + * walks the whole thing. + */ +describe('the autodial address list is bounded', () => { + const discovered = (i: number): string => `/ip4/198.51.100.${i % 254}/tcp/${9000 + i}/p2p/${PEER_ID}`; + const CONFIGURED = `/ip4/203.0.113.1/tcp/9090/p2p/${PEER_ID}`; + + function bareNetwork(configured: string[] = []) { + const network = Object.create(Network.prototype) as Network; + (network as any).bootstrapMultiaddrs = []; + (network as any).configuredBootstrapAddresses = new Set(configured.map(a => normalizeMultiaddrForCompare(a))); + for (const address of configured) (network as any).rememberBootstrapAddress(multiaddr(address)); + return network; + } + + const remember = (network: Network, count: number): void => { + for (let i = 0; i < count; i++) (network as any).rememberBootstrapAddress(multiaddr(discovered(i))); + }; + const addresses = (network: Network): string[] => (network as any).bootstrapMultiaddrs.map((m: { toString(): string }) => m.toString()); + + it('stops growing past the ceiling', () => { + const network = bareNetwork(); + remember(network, 600); + expect((network as any).bootstrapMultiaddrs).toHaveLength(512); + }); + + it('drops the oldest discovered entry, keeping the newest', () => { + const network = bareNetwork(); + remember(network, 600); + expect(addresses(network)).not.toContain(multiaddr(discovered(0)).toString()); + expect(addresses(network)).toContain(multiaddr(discovered(599)).toString()); + }); + + /** Configured entries are the user's way back into a network and are never evicted. */ + it('never drops a configured address to make room', () => { + const network = bareNetwork([CONFIGURED]); + remember(network, 600); + expect(addresses(network)).toContain(multiaddr(CONFIGURED).toString()); + }); +}); diff --git a/backend/tests/unit/protocol/peer-eviction.test.ts b/backend/tests/unit/protocol/peer-eviction.test.ts new file mode 100644 index 000000000..53a50961d --- /dev/null +++ b/backend/tests/unit/protocol/peer-eviction.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'bun:test'; +import { nextEvictionWindowStart, nextEvictionFailCount, shouldEvictUnreachablePeer } from '../../../src/protocol/network.ts'; + +/** The thresholds the production constants use, restated so a change to either is visible here. */ +const EVICT_FAILS = 6; +const EVICT_MIN_MS = 30 * 60_000; +const NOW = 1_700_000_000_000; + +describe('nextEvictionWindowStart', () => { + it('keeps the original start while this node can reach someone', () => { + const started = NOW - 20 * 60_000; + expect(nextEvictionWindowStart(true, started, NOW)).toBe(started); + }); + + it('starts the window at the first failure when there is no earlier one', () => { + expect(nextEvictionWindowStart(true, undefined, NOW)).toBe(NOW); + }); + + it('slides the window forward for every failure suffered while offline', () => { + // This is the whole point: an hour of failures during a local outage must + // not count towards the peer's unreachability. + const started = NOW - 60 * 60_000; + expect(nextEvictionWindowStart(false, started, NOW)).toBe(NOW); + }); +}); + +describe('shouldEvictUnreachablePeer', () => { + const gone = { reachable: true, failCount: EVICT_FAILS, unreachableForMs: EVICT_MIN_MS, configured: false }; + + it('evicts a peer that failed enough times over enough time', () => { + expect(shouldEvictUnreachablePeer(gone)).toBe(true); + }); + + it('never evicts while this node has no other connection', () => { + // A node that cannot reach anybody has no evidence about anybody. Without + // this, a laptop waking after an hour asleep would purge its whole peerStore. + expect(shouldEvictUnreachablePeer({ ...gone, reachable: false })).toBe(false); + }); + + it('never evicts a peer the operator configured by hand', () => { + // Configured bootstrap peers are user data: they keep their red status row + // through any outage rather than disappearing from the list. + expect(shouldEvictUnreachablePeer({ ...gone, configured: true })).toBe(false); + }); + + it('waits for both the count and the duration, not either', () => { + expect(shouldEvictUnreachablePeer({ ...gone, failCount: EVICT_FAILS - 1 })).toBe(false); + expect(shouldEvictUnreachablePeer({ ...gone, unreachableForMs: EVICT_MIN_MS - 1 })).toBe(false); + }); + + it('keeps evicting once both thresholds are exceeded', () => { + expect(shouldEvictUnreachablePeer({ ...gone, failCount: 50, unreachableForMs: 5 * EVICT_MIN_MS })).toBe(true); + }); +}); + +describe('the outage scenario the guard exists for', () => { + it('does not evict on the first failure after an hour offline', () => { + // Reproduces the reported defect: six failures accumulated while the host + // itself was offline, then the connection returns. Before the fix the stored + // window was an hour old, so this very first failure satisfied the duration + // test and purged the peer. + const windowStartedAt = NOW - 60 * 60_000; + + // Six failures during the outage: each one slides the window forward. + let start = windowStartedAt; + for (let failure = 1; failure <= EVICT_FAILS; failure++) start = nextEvictionWindowStart(false, start, NOW - (EVICT_FAILS - failure) * 1000); + + const firstFailureBackOnline = NOW; + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS + 1, unreachableForMs: firstFailureBackOnline - start, configured: false })).toBe(false); + }); + + it('still evicts a peer that stays unreachable while we are online', () => { + // The guard must not make eviction unreachable in practice — a genuinely + // dead peer is still removed once the failures accumulate with us online. + let start = nextEvictionWindowStart(true, undefined, NOW - EVICT_MIN_MS); + for (let failure = 2; failure <= EVICT_FAILS; failure++) start = nextEvictionWindowStart(true, start, NOW); + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS, unreachableForMs: NOW - start, configured: false })).toBe(true); + }); +}); + +describe('negative control', () => { + /** The pre-fix behaviour: the window ran from the first failure regardless of our own reachability. */ + function windowStartBeforeFix(previous: number | undefined, now: number): number { + return previous ?? now; + } + /** The pre-fix condition: count and duration only, with no notion of whether we were online. */ + function evictBeforeFix(failCount: number, unreachableForMs: number, configured: boolean): boolean { + return failCount >= EVICT_FAILS && unreachableForMs >= EVICT_MIN_MS && !configured; + } + + it('proves the scenario discriminates: the old logic evicts where the new one does not', () => { + // Same inputs as "does not evict on the first failure after an hour offline". + // If this assertion ever flips, the scenario stopped exercising the defect + // and the test above would pass for the wrong reason. + let start = NOW - 60 * 60_000; + for (let failure = 1; failure <= EVICT_FAILS; failure++) start = windowStartBeforeFix(start, NOW - (EVICT_FAILS - failure) * 1000); + + expect(evictBeforeFix(EVICT_FAILS + 1, NOW - start, false)).toBe(true); + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: EVICT_FAILS + 1, unreachableForMs: NOW - nextEvictionWindowStart(false, start, NOW), configured: false })).toBe(false); + }); +}); + +/** + * The backoff counter and the eviction counter answer different questions. The backoff + * must keep growing through a local outage so we stop hammering the dialer; eviction + * asks whether the PEER failed us, and a dial attempted with no connectivity of our own + * answers nothing. + */ +describe('nextEvictionFailCount', () => { + it('counts a failure that happened while we were online', () => { + expect(nextEvictionFailCount(true, 2)).toBe(3); + }); + + it('starts the count at one for a first online failure', () => { + expect(nextEvictionFailCount(true, undefined)).toBe(1); + }); + + it('resets the run when the failure happened during our own outage', () => { + expect(nextEvictionFailCount(false, 5)).toBe(0); + }); +}); + +describe('an outage must not bank failures towards eviction', () => { + /** Backoff caps at 10 min, so a 30 min window holds roughly three attempts. */ + const ONLINE_ATTEMPTS_IN_WINDOW = 3; + + it('does not evict on three online failures after a long offline run', () => { + let evictionFails: number | undefined; + for (let i = 0; i < 20; i++) evictionFails = nextEvictionFailCount(false, evictionFails); // hours offline + for (let i = 0; i < ONLINE_ATTEMPTS_IN_WINDOW; i++) evictionFails = nextEvictionFailCount(true, evictionFails); + + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: evictionFails!, unreachableForMs: 45 * 60_000, configured: false })).toBe(false); + }); + + it('still evicts once the peer really has failed enough times while we were online', () => { + let evictionFails: number | undefined; + for (let i = 0; i < 20; i++) evictionFails = nextEvictionFailCount(false, evictionFails); + for (let i = 0; i < 6; i++) evictionFails = nextEvictionFailCount(true, evictionFails); + + expect(shouldEvictUnreachablePeer({ reachable: true, failCount: evictionFails!, unreachableForMs: 45 * 60_000, configured: false })).toBe(true); + }); +}); diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 7a569ca05..dafcc56c9 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2020", "module": "esnext", - "lib": ["ES2020"], + "lib": ["ES2021"], "moduleResolution": "bundler", "allowImportingTsExtensions": true, "noEmit": true, diff --git a/frontend/src/pages/Settings/SettingsFactoryReset.svelte b/frontend/src/pages/Settings/SettingsFactoryReset.svelte index 32ead3162..323ea19a3 100644 --- a/frontend/src/pages/Settings/SettingsFactoryReset.svelte +++ b/frontend/src/pages/Settings/SettingsFactoryReset.svelte @@ -54,10 +54,14 @@ const res = await api.settings.factoryReset({ settings: resetSettings, identity: resetIdentity, downloads: resetDownloads, networks: resetNetworks, peers: resetPeers }); // Each category is wiped independently — one alert per category on the done page. const labelKey: Record = { settings: 'optionSettings', identity: 'optionIdentity', downloads: 'optionDownloads', networks: 'optionNetworks', peers: 'optionPeers' }; - resultAlerts = res.results.map(r => { + const categoryAlerts = res.results.map(r => { const category = tt('settings.factoryReset.' + labelKey[r.category]); return r.ok ? { type: 'info' as const, message: tt('settings.factoryReset.categoryDone', { category }) } : { type: 'error' as const, message: tt('settings.factoryReset.categoryFailed', { category, detail: r.detail ?? '' }) }; }); + // A failed prepare is why the wipes were skipped, a failed restart means the node + // is still down — both matter more than any single category outcome. + const failedPhase = (phase: 'prepare' | 'restart') => res.phases.filter(p => p.phase === phase && !p.ok).map(p => ({ type: 'error' as const, message: tt('settings.factoryReset.' + phase + 'Failed', { detail: p.detail ?? '' }) })); + resultAlerts = [...failedPhase('prepare'), ...categoryAlerts, ...failedPhase('restart')]; phase = 'done'; } catch (e) { // Transport-level failure — the reset never ran. Back to the form with the error. diff --git a/frontend/static/langs/cs.json b/frontend/static/langs/cs.json index df23e3c04..197c6f98d 100644 --- a/frontend/static/langs/cs.json +++ b/frontend/static/langs/cs.json @@ -454,7 +454,9 @@ "confirmMessage": "Opravdu vymazat vybrané kategorie? Soubory na disku zůstanou, ale tuto akci nelze vrátit zpět.", "error": "Obnova továrního nastavení selhala: {detail}", "categoryDone": "{category} – vyresetováno", - "categoryFailed": "{category} – selhalo: {detail}" + "categoryFailed": "{category} – selhalo: {detail}", + "prepareFailed": "Přenosy a síťový uzel se nepodařilo zastavit, proto nebylo vymazáno nic, co je používá: {detail}", + "restartFailed": "Síťový uzel nebyl znovu spuštěn: {detail}" }, "identity": { "title": "Identita", diff --git a/frontend/static/langs/en.json b/frontend/static/langs/en.json index d3a78b387..8d9ca45ac 100644 --- a/frontend/static/langs/en.json +++ b/frontend/static/langs/en.json @@ -454,7 +454,9 @@ "confirmMessage": "Wipe the selected categories? Files on disk are kept, but this action cannot be undone.", "error": "Factory reset failed: {detail}", "categoryDone": "{category} – reset", - "categoryFailed": "{category} – failed: {detail}" + "categoryFailed": "{category} – failed: {detail}", + "prepareFailed": "Transfers and the network node could not be stopped, so nothing that uses them was wiped: {detail}", + "restartFailed": "The network node was not started again: {detail}" }, "identity": { "title": "Identity", diff --git a/shared/src/index.ts b/shared/src/index.ts index fb4582163..317e4a140 100644 --- a/shared/src/index.ts +++ b/shared/src/index.ts @@ -214,11 +214,27 @@ export interface FactoryResetResult { detail?: string; } +/** The infrastructure steps run around the wipes. `prepare` stops the transfers and the + * node, `restart` brings them back — neither is a wipe, but both can fail in ways the + * user has to know about: a failed `prepare` means the destructive categories were not + * safe to run, a failed `restart` means the node is still down. */ +export type FactoryResetPhase = 'prepare' | 'restart'; + +/** Outcome of one factory-reset phase. */ +export interface FactoryResetPhaseResult { + phase: FactoryResetPhase; + ok: boolean; + /** Failure (or skip) reason when `ok` is false. */ + detail?: string; +} + /** Aggregate factory-reset response: `success` is true only when every selected - * category succeeded; `results` carries the per-category outcome. */ + * category AND every phase succeeded; `results` carries the per-category outcome and + * `phases` the prepare/restart outcome. */ export interface FactoryResetResponse { success: boolean; results: FactoryResetResult[]; + phases: FactoryResetPhaseResult[]; } // Dataset types (derived from ILISH entries that have a directory)