diff --git a/backend/src/api/lishnets.ts b/backend/src/api/lishnets.ts index bb49e343..eb7113d0 100644 --- a/backend/src/api/lishnets.ts +++ b/backend/src/api/lishnets.ts @@ -183,9 +183,14 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, try { const { stream } = await network.dialProtocolByPeerId(p.peerID, LISH_PROTOCOL); const client = new LISHClient(stream); - const lishs = await client.requestList(); - await client.close(); - return { lishs }; + try { + const lishs = await client.requestList(); + return { lishs }; + } finally { + // Close in finally so a throwing request (peer error, validation) cannot leak + // the stream; swallow close errors so they never mask the request error. + await client.close().catch(() => {}); + } } catch (error: any) { if (error instanceof CodedError) throw error; console.error(`[Peers] Failed to get LISH list from ${p.peerID.slice(0, 12)}:`, error.message?.slice(0, 120) ?? error); @@ -199,8 +204,13 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, const { stream } = await network.dialProtocolByPeerId(p.peerID, LISH_PROTOCOL); const client = new LISHClient(stream); const onProgress = (received: number, total: number): void => broadcast('lishnets:manifestProgress', { lishID: p.lishID, peerID: p.peerID, received, total } satisfies ManifestProgressEvent); - const manifest = await client.requestManifest(p.lishID, onProgress); - await client.close(); + let manifest; + try { + manifest = await client.requestManifest(p.lishID, onProgress); + } finally { + // Close in finally — a rejected manifest (validation, peer error) must not leak the stream. + await client.close().catch(() => {}); + } // Strip checksums from files and compute summary const files = (manifest.files ?? []).map(f => { const entry: { path: string; size: number; permissions?: string; modified?: string; created?: string } = { path: f.path, size: f.size }; @@ -238,8 +248,12 @@ export function initLISHnetsHandlers(networks: Networks, dataServer: DataServer, const { stream } = await network.dialProtocolByPeerId(p.peerID, LISH_PROTOCOL); const client = new LISHClient(stream); const onProgress = (received: number, total: number): void => broadcast('lishnets:manifestProgress', { lishID: p.lishID, peerID: p.peerID, received, total } satisfies ManifestProgressEvent); - manifest = await client.requestManifest(p.lishID, onProgress); - await client.close(); + try { + manifest = await client.requestManifest(p.lishID, onProgress); + } finally { + // Close in finally — a rejected manifest (validation, peer error) must not leak the stream. + await client.close().catch(() => {}); + } } catch (error: any) { if (error instanceof CodedError) throw error; console.error(`[Peers] Failed to add LISH ${p.lishID.slice(0, 8)} from ${p.peerID.slice(0, 12)}:`, error.message?.slice(0, 120) ?? error); diff --git a/backend/src/api/lishs.ts b/backend/src/api/lishs.ts index db689f72..05edc81a 100644 --- a/backend/src/api/lishs.ts +++ b/backend/src/api/lishs.ts @@ -1,5 +1,5 @@ import { type DataServer } from '../lish/data-server.ts'; -import { type ILISH, type IStoredLISH, type ILISHDetail, type ILISHListResult, type SuccessResponse, type CreateLISHResponse, type ImportLISHResponse, type LISHSortField, type SortOrder, type CompressionAlgorithm, DEFAULT_ALGO, sanitizeFilename, validateLISHStructure, CodedError, ErrorCodes, productName } from '@shared'; +import { type ILISH, type IStoredLISH, type ILISHDetail, type ILISHListResult, type SuccessResponse, type CreateLISHResponse, type ImportLISHResponse, type LISHSortField, type SortOrder, type CompressionAlgorithm, DEFAULT_ALGO, sanitizeFilename, validateLISHStructure, formatSizeOverLimit, CodedError, ErrorCodes, productName } from '@shared'; import { createLISH, exportLISHToFile, importLISHFromFile, parseLISHFromJSON, runVerification } from '../lish/lish.ts'; import { DEFAULT_CHUNK_SIZE } from '@shared'; import { Utils } from '../utils.ts'; @@ -202,8 +202,11 @@ export function initLISHsHandlers(dataServer: DataServer, emit: EmitFn, broadcas const chunkSize = p.chunkSize ?? DEFAULT_CHUNK_SIZE; // Reject overly large chunkSize before the (potentially long) hashing pass. const maxChunkSize: number = settings.get('network.maxChunkSize') ?? DEFAULT_MAX_CHUNK_SIZE; - if (typeof chunkSize !== 'number' || !Number.isFinite(chunkSize) || chunkSize <= 0) throw new CodedError(ErrorCodes.LISH_INVALID_CHUNK_SIZE, String(chunkSize)); - if (chunkSize > maxChunkSize) throw new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, `${chunkSize} > ${maxChunkSize}`); + // Match validateLISHStructure's contract (integer chunkSize) so a LISH this + // version creates is always one it can also import — a fractional size would + // pass creation/export but be rejected on import. + if (typeof chunkSize !== 'number' || !Number.isInteger(chunkSize) || chunkSize <= 0) throw new CodedError(ErrorCodes.LISH_INVALID_CHUNK_SIZE, String(chunkSize)); + if (chunkSize > maxChunkSize) throw new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, formatSizeOverLimit(chunkSize, maxChunkSize)); const threads = p.threads ?? 0; // 0 = all CPU threads const minifyJSON = p.minifyJSON ?? false; const compress = p.compress ?? false; diff --git a/backend/src/api/settings.ts b/backend/src/api/settings.ts index c3a39405..228f6201 100644 --- a/backend/src/api/settings.ts +++ b/backend/src/api/settings.ts @@ -1,7 +1,7 @@ import { type Settings, type SettingsData } from '../settings.ts'; import { applyNetworkLimits } from '../protocol/network-limits.ts'; import { Utils } from '../utils.ts'; -import { type CompressionAlgorithm, type SuccessResponse, type ISettingsImportResult, CodedError, ErrorCodes } from '@shared'; +import { type CompressionAlgorithm, type SuccessResponse, type ISettingsImportResult, CodedError, ErrorCodes, minMessageSizeFor } from '@shared'; const assert = Utils.assertParams; const ALLOWED_ROOT_KEYS = new Set(['language', 'ui', 'audio', 'storage', 'network', 'system', 'export', 'input']); @@ -54,6 +54,22 @@ export function initSettingsHandlers(settings: Settings): SettingsHandlers { return settings.get(p.path); } + /** + * Push network limits into the protocol layer, first repairing a stored message-size + * limit that could not carry one chunk. applyNetworkLimits() enforces the same floor at + * runtime; persisting it here as well keeps the settings screen from showing a value the + * protocol layer silently overrides. + * + * Used by the paths that write arbitrary values — the WS `settings.set` call and the + * settings import. Reset and factory reset write the defaults, which satisfy the floor + * by construction, so they call applyNetworkLimits() directly. + */ + async function persistAndApplyNetworkLimits(): Promise { + const floor = minMessageSizeFor(settings.get().network.maxChunkSize); + if (settings.get().network.maxMessageSize < floor) await settings.set('network.maxMessageSize', floor); + applyNetworkLimits(settings.get().network); + } + async function set(p: { path: string; value: any }): Promise { assert(p, ['path', 'value']); // Confine writes to known top-level settings groups. This rejects unknown @@ -64,7 +80,7 @@ export function initSettingsHandlers(settings: Settings): SettingsHandlers { await settings.set(p.path, p.value); // Re-push all runtime limits on any network write (idempotent). Path-by-path // matching used to miss whole-object writes such as path === 'network'. - if (rootKey === 'network') applyNetworkLimits(settings.get().network); + if (rootKey === 'network') await persistAndApplyNetworkLimits(); return true; } @@ -147,7 +163,9 @@ export function initSettingsHandlers(settings: Settings): SettingsHandlers { skipped.push(entry.path); } } - applyNetworkLimits(settings.get().network); + // An imported file can carry a message limit below the chunk limit — repair it here + // too, not just on interactive writes. + await persistAndApplyNetworkLimits(); console.log(`✓ Settings restored: ${applied} applied, ${skipped.length} skipped`); return { applied, skipped }; } diff --git a/backend/src/protocol/chunk-downloader.ts b/backend/src/protocol/chunk-downloader.ts index 1444698d..0245632f 100644 --- a/backend/src/protocol/chunk-downloader.ts +++ b/backend/src/protocol/chunk-downloader.ts @@ -1,5 +1,5 @@ import { Mutex } from 'async-mutex'; -import { type ChunkID, type IStoredLISH, type LISHid, ErrorCodes } from '@shared'; +import { type ChunkID, type IStoredLISH, type LISHid, ErrorCodes, expectedChunkLength } from '@shared'; import { DataServer, type MissingChunk } from '../lish/data-server.ts'; import { LISHClient, type HaveChunks } from './lish-protocol.ts'; import { downloadLimiter } from './speed-limiter.ts'; @@ -299,18 +299,26 @@ export class ChunkDownloader { } continue; } - // Verify chunk integrity before writing + // Reject bad chunk data before writing. The manifest fixes each chunk's exact byte + // length (a file's last chunk may be shorter than chunkSize); checking length before + // the O(n) hash stops a peer from forcing us to hash oversized payloads (bounded only + // by maxMessageSize) and rejects malformed data early. const data = result.data; - const hasher = new Bun.CryptoHasher(lish.checksumAlgo as any); - hasher.update(data); - const actualHash = hasher.digest('hex'); - if (actualHash !== chunk.chunkID) { + const expectedLen = expectedChunkLength(lish, chunk.fileIndex, chunk.chunkIndex); + let rejectReason: string | null = expectedLen >= 0 && data.length !== expectedLen ? `wrong length: expected ${expectedLen}B, got ${data.length}B` : null; + if (!rejectReason) { + const hasher = new Bun.CryptoHasher(lish.checksumAlgo as any); + hasher.update(data); + const actualHash = hasher.digest('hex'); + if (actualHash !== chunk.chunkID) rejectReason = `bad hash: expected ${chunk.chunkID.slice(0, 12)}, got ${actualHash.slice(0, 12)}`; + } + if (rejectReason) { const count = (corruptCount.get(peerID) ?? 0) + 1; corruptCount.set(peerID, count); - console.log(`[DL] Corrupt chunk from ${peerID.slice(0, 12)}: expected ${chunk.chunkID.slice(0, 12)}, got ${actualHash.slice(0, 12)} (${count}/${ChunkDownloader.MAX_CORRUPT_CHUNKS})`); + console.log(`[DL] Rejected chunk from ${peerID.slice(0, 12)} (${rejectReason}) (${count}/${ChunkDownloader.MAX_CORRUPT_CHUNKS})`); await requeueChunk(chunk); if (count >= ChunkDownloader.MAX_CORRUPT_CHUNKS) { - console.log(`[DL] Peer ${peerID.slice(0, 12)} banned: ${count} corrupt chunks`); + console.log(`[DL] Peer ${peerID.slice(0, 12)} banned: ${count} bad chunks`); await peerManager.removeAwait(peerID, 'ban'); break; } diff --git a/backend/src/protocol/downloader.ts b/backend/src/protocol/downloader.ts index 922dde56..90695064 100644 --- a/backend/src/protocol/downloader.ts +++ b/backend/src/protocol/downloader.ts @@ -390,11 +390,32 @@ export class Downloader { // Phase 1: fetch manifest from a peer if needed if (this.state === 'awaiting-manifest') { if (this.peerManager.size() === 0) return; - for (const [, client] of this.peerManager.entries()) { + // LIVE iteration on purpose (Map iterators tolerate deletes and visit entries + // added mid-loop): a peer joining via HAVE while we await another's manifest + // gets its turn in THIS pass — its doWork() trigger no-ops on the locked mutex. + for (const [peerID, client] of this.peerManager.entries()) { let manifest: import('@shared').IStoredLISH | null = null; try { manifest = await client.requestManifest(this.lishID); } catch (error: any) { + if (error instanceof CodedError && error.code === ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE) { + // The peer delivered a well-formed manifest for the LISH we asked for and it + // declares a chunk size above our limit. Chunk size is a property of the LISH + // itself, so every honest peer serves the same value — asking the rest only + // makes the user watch each peer fail in turn before the same error appears. + // Surface it now and stop — unless the download was torn down while we + // awaited the manifest, in which case there is no state left to fail. + this.peerManager.remove(peerID, 'drop'); + if (!this.destroyed) this.setError(error.code, error.detail); + return; + } + // A structurally malformed manifest (mapped to PEER_INVALID_REQUEST) is this + // peer's fault — keeping it would leave the download stuck asking the same + // bad peer forever while discovery skips it as "connected". + if (error instanceof CodedError && error.code === ErrorCodes.PEER_INVALID_REQUEST) { + this.peerManager.remove(peerID, 'drop'); + continue; + } console.warn(`[DL] Manifest request failed: ${error.message?.slice(0, 120) ?? error}`); } if (manifest && manifest.files && manifest.files.length > 0) { @@ -571,6 +592,22 @@ export class Downloader { try { manifest = await probeClient.requestManifest(this.lishID); } catch (error: any) { + // Any manifest error (unreachable, malformed) → drop this peer and let another + // serve it, except over-limit which is terminal for the whole LISH — but only + // while we are still looking for a manifest. Probing also runs mid-download + // purely to find more peers, and there requestManifest is just a "do you have + // this LISH?" test whose answer is discarded; failing the transfer on it would + // hand any peer on the topic a way to kill a healthy download. + if (this.needsManifest && error instanceof CodedError && error.code === ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE) { + // Same reasoning as the connected-peer loop: a delivered manifest that is over + // the limit answers the question for the whole LISH, so stop probing the rest. + // close() must not throw past this point — the outer catch would swallow the + // verdict, log it as an unreachable peer and let the probe loop carry on. + this.peerManager.remove(peerID, 'drop'); + await probeClient.close().catch(() => {}); + if (!this.destroyed) this.setError(error.code, error.detail); + return; + } console.debug(`[DL] probe ${peerID.slice(0, 12)}: manifest error ${error.code ?? error.message?.slice(0, 60) ?? error}`); this.peerManager.remove(peerID, 'drop'); } diff --git a/backend/src/protocol/lish-protocol.ts b/backend/src/protocol/lish-protocol.ts index a940b6d2..edd6d90d 100644 --- a/backend/src/protocol/lish-protocol.ts +++ b/backend/src/protocol/lish-protocol.ts @@ -1,8 +1,8 @@ import { decode as lpDecode } from 'it-length-prefixed'; import { encode as lpEncode } from 'it-length-prefixed'; import { type Stream } from '@libp2p/interface'; -import { type LISHid, type ChunkID, type ErrorCode, ErrorCodes, CodedError } from '@shared'; -import { DEFAULT_MAX_MESSAGE_SIZE } from '../settings.ts'; +import { type LISHid, type ChunkID, type ErrorCode, ErrorCodes, CodedError, validateLISHStructure } from '@shared'; +import { DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_MAX_CHUNK_SIZE } from '../settings.ts'; import { type DataServer } from '../lish/data-server.ts'; import { Uint8ArrayList } from 'uint8arraylist'; import { uploadLimiter } from './speed-limiter.ts'; @@ -26,6 +26,21 @@ export function getMaxMessageSize(): number { return maxMessageSize; } +/** + * Maximum chunk size accepted in a manifest received from a peer, in bytes. + * Mirrors the `network.maxChunkSize` bound enforced on locally created/imported LISHs so a + * malicious or malformed manifest can't push an oversized chunk size into the app. Read live + * on every manifest so settings changes take effect without a peer restart. + */ +let maxChunkSize: number = DEFAULT_MAX_CHUNK_SIZE; +export function setMaxChunkSize(size: number): void { + if (typeof size === 'number' && Number.isFinite(size) && size > 0) maxChunkSize = size; +} + +export function getMaxChunkSize(): number { + return maxChunkSize; +} + export type LISHRequest = LISHGetChunkRequest | LISHGetLishRequest | LISHGetLishsRequest | LISHAnnounceHaveRequest | LISHSearchResultRequest; export interface LISHGetChunkRequest { type?: 'getChunk'; @@ -211,6 +226,23 @@ export class LISHClient { const response = this.parseResponse(responseData, `getLish ${lishID}`); if ('error' in response) throw new CodedError(response.error, lishID); if (!('manifest' in response)) throw new CodedError(ErrorCodes.PEER_INVALID_REQUEST, `getLish ${lishID}: missing manifest`); + // The manifest must be for the LISH we asked for — a peer returning a different id + // would let a spoofing peer win the fallback loop and could import the wrong LISH + // under the requested id. Treat a mismatch as this peer's fault so fallback tries the next. + if (response.manifest?.id !== lishID) throw new CodedError(ErrorCodes.PEER_INVALID_REQUEST, `getLish ${lishID}: manifest id mismatch (${String(response.manifest?.id)})`); + // A manifest from the network is untrusted input — validate chunk-size bounds and + // manifest consistency before it can reach any caller (DB persist / import / probe). + try { + validateLISHStructure(response.manifest, maxChunkSize); + } catch (e) { + // A structurally malformed manifest is this peer's fault — surface it as a peer + // protocol error so fallback loops move on to the next peer. An over-limit + // chunkSize is a property of the LISH itself (every honest peer serves the same + // manifest), so it stays a terminal local error. + if (e instanceof CodedError && e.code !== ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE) throw new CodedError(ErrorCodes.PEER_INVALID_REQUEST, `getLish ${lishID}: ${e.message}`); + throw e; + } + // Emitted only after validation passes — a rejected manifest must not flash a full bar. if (total > 0) safeEmit(total, total); return response.manifest; } finally { diff --git a/backend/src/protocol/network-limits.ts b/backend/src/protocol/network-limits.ts index 70a2f48d..aaa148e3 100644 --- a/backend/src/protocol/network-limits.ts +++ b/backend/src/protocol/network-limits.ts @@ -1,6 +1,7 @@ import { type SettingsData } from '../settings.ts'; +import { minMessageSizeFor } from '@shared'; import { Downloader } from './downloader.ts'; -import { setMaxUploadSpeed, setMaxUploadPeersPerLISH, setMaxMessageSize } from './lish-protocol.ts'; +import { setMaxUploadSpeed, setMaxUploadPeersPerLISH, setMaxMessageSize, setMaxChunkSize } from './lish-protocol.ts'; import { setMaxDownloadPeersPerLISH } from './peer-manager.ts'; /** @@ -15,5 +16,10 @@ export function applyNetworkLimits(net: SettingsData['network']): void { setMaxUploadSpeed(net.maxUploadSpeed); setMaxDownloadPeersPerLISH(net.maxDownloadPeersPerLISH); setMaxUploadPeersPerLISH(net.maxUploadPeersPerLISH); - setMaxMessageSize(net.maxMessageSize); + // A message limit at or below the chunk limit would reject every chunk on arrival, so + // the chunk limit wins and the message limit is lifted over it. Enforced here rather + // than at each writer: startup, WS API set/reset/import and factory reset all pass + // through this function, so no path can install an unusable pair. + setMaxMessageSize(Math.max(net.maxMessageSize, minMessageSizeFor(net.maxChunkSize))); + setMaxChunkSize(net.maxChunkSize); } diff --git a/backend/tests/unit/protocol/chunk-downloader-loop.test.ts b/backend/tests/unit/protocol/chunk-downloader-loop.test.ts index 68c0b46c..327bcee9 100644 --- a/backend/tests/unit/protocol/chunk-downloader-loop.test.ts +++ b/backend/tests/unit/protocol/chunk-downloader-loop.test.ts @@ -222,6 +222,65 @@ describe('ChunkDownloader peerLoop — partial seeder behavior', () => { expect(ds.downloadedChunks.has(slowChunkID)).toBe(true); }, 15000); + it('rejects a wrong-length chunk before hashing and completes from a healthy peer', async () => { + // The bad peer serves a truncated payload for the only chunk. The length check + // (not the hash) must reject it, the chunk is requeued and the healthy peer + // finishes the download. + const { missing, data } = makeChunks(1); + const onlyID = missing[0]!.chunkID; + const truncated = data.get(onlyID)!.subarray(0, CHUNK_SIZE / 2); + const ds = new FakeDataServer(missing); + const pm = new PeerManager(); + const bad = new ScriptedClient(new Map([[onlyID, truncated]])); + const good = new ScriptedClient(new Map([[onlyID, data.get(onlyID)!]]), 100); + const cd = makeDownloader(ds, pm, 1); + pm.tryAdd('peer-bad-length0', bad as never, 'DIRECT'); + pm.tryAdd('peer-good-00000', good as never, 'DIRECT'); + + const logs: string[] = []; + const origLog = console.log; + console.log = (...args: unknown[]) => logs.push(args.join(' ')); + try { + await cd.run(); + } finally { + console.log = origLog; + } + + expect(ds.downloadedChunks.has(onlyID)).toBe(true); + expect(logs.some(l => l.includes('Rejected chunk') && l.includes('wrong length'))).toBe(true); + }, 15000); + + it('bans a peer that keeps serving wrong-length chunks', async () => { + // Every reply from the bad peer is oversized. After MAX_CORRUPT_CHUNKS + // rejections the peer must be banned instead of being asked forever. + const { missing, data } = makeChunks(4); + const replies = new Map(); + for (const c of missing) { + const oversized = new Uint8Array(CHUNK_SIZE + 1); + oversized.set(data.get(c.chunkID)!); + replies.set(c.chunkID, oversized); + } + const ds = new FakeDataServer(missing); + const pm = new PeerManager(); + const bad = new ScriptedClient(replies); + const cd = makeDownloader(ds, pm, 4); + pm.tryAdd('peer-oversize00', bad as never, 'DIRECT'); + + const logs: string[] = []; + const origLog = console.log; + console.log = (...args: unknown[]) => logs.push(args.join(' ')); + try { + await cd.run(); + } finally { + console.log = origLog; + } + + expect(ds.downloadedChunks.size).toBe(0); + expect(logs.some(l => l.includes('banned') && l.includes('bad chunks'))).toBe(true); + // Banned after 3 rejected chunks — never probed the 4th. + expect(bad.requests.length).toBe(3); + }, 15000); + it('retries a cached miss when a connected peer announces the chunk in a new HAVE', async () => { const { missing, data } = makeChunks(2); const newlyAvailableID = missing[0]!.chunkID; diff --git a/backend/tests/unit/protocol/downloader-probe-oversized.test.ts b/backend/tests/unit/protocol/downloader-probe-oversized.test.ts new file mode 100644 index 00000000..d07c0645 --- /dev/null +++ b/backend/tests/unit/protocol/downloader-probe-oversized.test.ts @@ -0,0 +1,99 @@ +/** + * Peer discovery keeps probing while a download runs, and there `requestManifest` is only a + * "do you have this LISH?" test — the answer is thrown away because the manifest is already + * imported. A peer answering that probe with an over-limit manifest must therefore not be + * able to fail a healthy transfer; the verdict belongs to the manifest-fetch path only. + * + * Uses the real LISHClient over a canned stream (same trick as lish-protocol.test.ts) rather + * than mocking the module — a module mock leaks into every other test file in the run. + */ +import { test, expect, afterEach } from 'bun:test'; +import { encode as lpEncode } from 'it-length-prefixed'; +import { Downloader } from '../../../src/protocol/downloader.ts'; +import { setMaxChunkSize } from '../../../src/protocol/lish-protocol.ts'; +import { encode as codecEncode } from '../../../src/protocol/codec.ts'; +import { DEFAULT_MAX_CHUNK_SIZE } from '../../../src/settings.ts'; +import { ErrorCodes, type IStoredLISH } from '@shared'; + +const CHUNK_LIMIT = 1024 * 1024; +const priv = (o: unknown): Record => o as unknown as Record; + +/** Manifest declaring a chunk size well past the limit set below. */ +function oversizedManifest(): IStoredLISH { + const chunkSize = 8 * CHUNK_LIMIT; + return { + id: 'test-probe-lish', + created: new Date().toISOString(), + chunkSize, + checksumAlgo: 'sha256', + files: [{ path: 'a.bin', size: chunkSize, checksums: ['h1'] }], + } as IStoredLISH; +} + +/** Stream stub that answers any request with one length-prefixed manifest frame. */ +function cannedStream(): any { + const frame = lpEncode.single(codecEncode({ manifest: oversizedManifest() })).subarray(); + async function* source() { + yield frame; + } + return { status: 'open', send() {}, close: async () => {}, abort() {}, [Symbol.asyncIterator]: source }; +} + +/** Network stub listing one topic peer whose dial hands back the canned stream. */ +class ProbeNetwork { + subscribe(): void {} + unsubscribeHandler(): void {} + async broadcast(): Promise {} + getTopicPeers(): string[] { + return ['peer-probe-0001']; + } + async dialProtocolByPeerId(): Promise<{ stream: unknown; connectionType: string }> { + return { stream: cannedStream(), connectionType: 'direct' }; + } + isRunning(): boolean { + return true; + } +} + +function makeDownloader(): any { + const ds = { + getMissingChunks: () => [], + getAllChunkCount: () => 2, + add: () => {}, + isChunkDownloaded: () => false, + }; + const dl = new Downloader('/tmp/dl-probe', new ProbeNetwork() as never, ds as never, 'net-001'); + priv(dl)['lishID'] = 'test-probe-lish'; + return dl; +} + +afterEach(() => { + setMaxChunkSize(DEFAULT_MAX_CHUNK_SIZE); +}); + +test('a probe answered with an over-limit manifest does not fail a running download', async () => { + setMaxChunkSize(CHUNK_LIMIT); + const dl = makeDownloader(); + // Manifest already fetched and validated — the probe only looks for more peers here. + priv(dl)['state'] = 'downloading'; + priv(dl)['needsManifest'] = false; + priv(dl)['lish'] = { id: 'test-probe-lish', chunkSize: 1024, files: [] }; + + await priv(dl)['probeTopicPeers'](); + + expect(priv(dl)['state']).toBe('downloading'); + expect(priv(dl)['errorCode']).toBeUndefined(); +}); + +test('the same answer is terminal while the manifest is still missing', async () => { + setMaxChunkSize(CHUNK_LIMIT); + const dl = makeDownloader(); + priv(dl)['state'] = 'awaiting-manifest'; + priv(dl)['needsManifest'] = true; + priv(dl)['lish'] = null; + + await priv(dl)['probeTopicPeers'](); + + expect(priv(dl)['state']).toBe('error'); + expect(priv(dl)['errorCode']).toBe(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE); +}); diff --git a/backend/tests/unit/protocol/downloader.test.ts b/backend/tests/unit/protocol/downloader.test.ts index 420c207e..f4af0dff 100644 --- a/backend/tests/unit/protocol/downloader.test.ts +++ b/backend/tests/unit/protocol/downloader.test.ts @@ -21,6 +21,8 @@ interface ChunkVerifyResult { class MockLISHClient { requestChunkResult: ChunkResult = new Uint8Array(1024).fill(0xff); requestManifestResult: ManifestResult = null; + requestManifestError: Error | null = null; + requestManifestCalls = 0; closeCalled = false; haveChunks: 'all' | ChunkID[] = 'all'; @@ -30,6 +32,8 @@ class MockLISHClient { } async requestManifest(_lishID: LISHid): Promise { + this.requestManifestCalls++; + if (this.requestManifestError) throw this.requestManifestError; return this.requestManifestResult; } @@ -1295,3 +1299,128 @@ describe('Downloader — inline ENOSPC retry', () => { expect(pc.writeResolvers.length).toBe(0); }); }); + +// --------------------------------------------------------------------------- +// doWork Phase 1 — over-limit manifest handling across multiple peers +// --------------------------------------------------------------------------- + +describe('Downloader – oversized manifest across peers', () => { + function awaitingManifestDownloader(ds: MockDataServer): Downloader { + const dl = new Downloader('/tmp/dl-oversized', new MockNetwork() as never, ds as never, 'net-001'); + const p = priv(dl); + p['state'] = 'awaiting-manifest'; + p['needsManifest'] = true; + p['lish'] = null; + p['lishID'] = 'test-oversized-lish'; + return dl; + } + + function oversizedClient(): MockLISHClient { + const c = new MockLISHClient(); + c.requestManifestError = new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, '4.00 MB > 1.00 MB'); + return c; + } + + it('stops at the first over-limit manifest without asking the remaining peers', async () => { + const ds = new MockDataServer(); + const dl = awaitingManifestDownloader(ds); + const peers = (priv(dl)['peerManager'] as { peers: Map }).peers; + const second = new MockLISHClient(); + second.requestManifestResult = makeLISH(); + peers.set('peer-oversized-1', oversizedClient()); // first peer answers: chunk size over limit + peers.set('peer-valid-00001', second); // must never be asked + + await dl.doWork(); + + expect(priv(dl)['state']).toBe('error'); + expect(priv(dl)['errorCode']).toBe(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE); + expect(ds.addedLishs.length).toBe(0); // nothing imported + expect(peers.has('peer-oversized-1')).toBe(false); // the answering peer was dropped + expect(second.requestManifestCalls ?? 0).toBe(0); // the rest were left alone + }); + + it('surfaces the terminal error when the only peer returns an over-limit manifest', async () => { + const ds = new MockDataServer(); + const dl = awaitingManifestDownloader(ds); + const peers = (priv(dl)['peerManager'] as { peers: Map }).peers; + peers.set('peer-oversized-1', oversizedClient()); + + await dl.doWork(); + + expect(ds.addedLishs.length).toBe(0); // nothing imported + expect(peers.size).toBe(0); // the over-limit peer dropped + expect(priv(dl)['state']).toBe('error'); + expect(priv(dl)['errorCode']).toBe(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE); + }); +}); + +describe('Downloader – malformed manifest peer handling', () => { + it('drops a peer whose manifest is malformed and imports from the next peer', async () => { + const ds = new MockDataServer(); + const dl = new Downloader('/tmp/dl-malformed', new MockNetwork() as never, ds as never, 'net-001'); + const p = priv(dl); + p['state'] = 'awaiting-manifest'; + p['needsManifest'] = true; + p['lish'] = null; + p['lishID'] = 'test-malformed-lish'; + const peers = (priv(dl)['peerManager'] as { peers: Map }).peers; + const bad = new MockLISHClient(); + bad.requestManifestError = new CodedError(ErrorCodes.PEER_INVALID_REQUEST, 'getLish: LISH_INVALID_MANIFEST'); + const good = new MockLISHClient(); + good.requestManifestResult = makeLISH(); + peers.set('peer-malformed-1', bad); + peers.set('peer-valid-00001', good); + + await dl.doWork(); + + expect(ds.addedLishs.length).toBe(1); // imported from the valid peer + expect(peers.has('peer-malformed-1')).toBe(false); // bad peer dropped, not stuck + }); +}); + +describe('Downloader – peer-fault manifest failures are not terminal', () => { + function awaitingDl(ds: MockDataServer): Downloader { + const dl = new Downloader('/tmp/dl-mixed', new MockNetwork() as never, ds as never, 'net-001'); + const p = priv(dl); + p['state'] = 'awaiting-manifest'; + p['needsManifest'] = true; + p['lish'] = null; + p['lishID'] = 'test-mixed-lish'; + return dl; + } + + it('malformed and unreachable peers keep the download awaiting discovery', async () => { + const ds = new MockDataServer(); + const dl = awaitingDl(ds); + const peers = (priv(dl)['peerManager'] as { peers: Map }).peers; + const malformed = new MockLISHClient(); + malformed.requestManifestError = new CodedError(ErrorCodes.PEER_INVALID_REQUEST, 'getLish: malformed'); + const unreachable = new MockLISHClient(); + unreachable.requestManifestError = new CodedError(ErrorCodes.PEER_UNREACHABLE, 'test-mixed-lish'); + peers.set('peer-malformed-1', malformed); + peers.set('peer-unreach-001', unreachable); + + await dl.doWork(); + + // Neither failure says anything about the LISH itself — keep awaiting discovery. + expect(priv(dl)['state']).toBe('awaiting-manifest'); + expect(priv(dl)['errorCode']).toBeUndefined(); + }); + + it('an over-limit peer is terminal even when another peer failed for its own reason first', async () => { + const ds = new MockDataServer(); + const dl = awaitingDl(ds); + const peers = (priv(dl)['peerManager'] as { peers: Map }).peers; + const malformed = new MockLISHClient(); + malformed.requestManifestError = new CodedError(ErrorCodes.PEER_INVALID_REQUEST, 'getLish: malformed'); + const oversized = new MockLISHClient(); + oversized.requestManifestError = new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, '4.00 MB > 1.00 MB'); + peers.set('peer-malformed-1', malformed); + peers.set('peer-oversized-1', oversized); + + await dl.doWork(); + + expect(priv(dl)['state']).toBe('error'); + expect(priv(dl)['errorCode']).toBe(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE); + }); +}); diff --git a/backend/tests/unit/protocol/lish-protocol.test.ts b/backend/tests/unit/protocol/lish-protocol.test.ts index 24257809..c056a848 100644 --- a/backend/tests/unit/protocol/lish-protocol.test.ts +++ b/backend/tests/unit/protocol/lish-protocol.test.ts @@ -1,6 +1,9 @@ -import { describe, it, expect, beforeEach } from 'bun:test'; -import { disableUpload, enableUpload, isUploadDisabled, getEnabledUploads, getActiveUploads, setUploadBroadcast, setMaxUploadSpeed, resetUploadState, type LISHGetChunkResponse } from '../../../src/protocol/lish-protocol.ts'; +import { describe, it, expect, beforeEach, afterEach } from 'bun:test'; +import { disableUpload, enableUpload, isUploadDisabled, getEnabledUploads, getActiveUploads, setUploadBroadcast, setMaxUploadSpeed, resetUploadState, LISHClient, setMaxChunkSize, type LISHGetChunkResponse } from '../../../src/protocol/lish-protocol.ts'; import { encode as codecEncode, decode as codecDecode } from '../../../src/protocol/codec.ts'; +import { encode as lpEncode } from 'it-length-prefixed'; +import { DEFAULT_MAX_CHUNK_SIZE } from '../../../src/settings.ts'; +import { ErrorCodes, type IStoredLISH } from '@shared'; // --------------------------------------------------------------------------- // Helpers @@ -364,3 +367,61 @@ describe('lish-protocol – msgpack chunk encoding', () => { expect(parsed.error).toBe('PEER_CHUNK_NOT_FOUND'); }); }); + +// --------------------------------------------------------------------------- +// requestManifest — validation of manifests received from peers +// --------------------------------------------------------------------------- + +describe('LISHClient.requestManifest – manifest validation', () => { + /** + * Minimal fake libp2p Stream that replays a single pre-built manifest response frame. + * `send()` is a no-op (requestManifest only writes the request); iteration yields the + * length-prefixed response the decoder reads back. + */ + function fakeStream(manifest: unknown): any { + const frame = lpEncode.single(codecEncode({ manifest })).subarray(); + async function* source() { + yield frame; + } + return { status: 'open', send() {}, close: async () => {}, [Symbol.asyncIterator]: source }; + } + + function makeManifest(chunkSize: number): IStoredLISH { + return { + id: 'lish-manifest-test', + created: new Date().toISOString(), + chunkSize, + checksumAlgo: 'sha256', + files: [{ path: 'a.bin', size: chunkSize, checksums: ['h1'] }], + }; + } + + afterEach(() => { + setMaxChunkSize(DEFAULT_MAX_CHUNK_SIZE); + }); + + it('rejects a manifest whose chunkSize exceeds the configured maximum', async () => { + setMaxChunkSize(1024 * 1024); + const client = new LISHClient(fakeStream(makeManifest(2 * 1024 * 1024))); + await expect(client.requestManifest('lish-manifest-test')).rejects.toMatchObject({ code: ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE }); + }); + + it('accepts a manifest whose chunkSize is within the maximum', async () => { + setMaxChunkSize(1024 * 1024); + const client = new LISHClient(fakeStream(makeManifest(1024))); + const manifest = await client.requestManifest('lish-manifest-test'); + expect(manifest.chunkSize).toBe(1024); + }); + + it('maps a structurally malformed manifest to a retryable peer error', async () => { + // Garbage from one peer must not abort peer-fallback loops — only PEER_* codes retry. + const client = new LISHClient(fakeStream({ ...makeManifest(1024), chunkSize: -5 })); + await expect(client.requestManifest('lish-manifest-test')).rejects.toMatchObject({ code: ErrorCodes.PEER_INVALID_REQUEST }); + }); + + it('rejects a manifest whose id does not match the requested LISH', async () => { + // A spoofing peer answering with a different LISH must not win the fallback. + const client = new LISHClient(fakeStream({ ...makeManifest(1024), id: 'some-other-lish' })); + await expect(client.requestManifest('lish-manifest-test')).rejects.toMatchObject({ code: ErrorCodes.PEER_INVALID_REQUEST }); + }); +}); diff --git a/backend/tests/unit/protocol/manifest-progress.test.ts b/backend/tests/unit/protocol/manifest-progress.test.ts index ef4f5f34..e02b8d7e 100644 --- a/backend/tests/unit/protocol/manifest-progress.test.ts +++ b/backend/tests/unit/protocol/manifest-progress.test.ts @@ -28,10 +28,16 @@ function buildManifestFrame(manifest: unknown): { frame: Uint8Array; dataLen: nu return { frame, dataLen: data.length }; } +// Structurally valid manifest — requestManifest now runs validateLISHStructure on +// every received manifest, so the fixture must satisfy it (chunkSize, matching +// checksum counts) or the progress path under test would never be reached. const MANIFEST = { id: 'lish-progress-test', name: 'progress', - files: Array.from({ length: 20 }, (_, i) => ({ path: `dir/file-${i}.bin`, size: 1000 + i })), + created: '2026-01-01T00:00:00Z', + chunkSize: 1024, + checksumAlgo: 'sha256', + files: Array.from({ length: 20 }, (_, i) => ({ path: `dir/file-${i}.bin`, size: 1000 + i, checksums: [`checksum-${i}`] })), }; // --------------------------------------------------------------------------- diff --git a/backend/tests/unit/protocol/network-limits.test.ts b/backend/tests/unit/protocol/network-limits.test.ts index 55b62b9b..2e92ad8b 100644 --- a/backend/tests/unit/protocol/network-limits.test.ts +++ b/backend/tests/unit/protocol/network-limits.test.ts @@ -1,8 +1,8 @@ import { describe, it, expect, afterAll } from 'bun:test'; import { applyNetworkLimits } from '../../../src/protocol/network-limits.ts'; -import { getMaxMessageSize } from '../../../src/protocol/lish-protocol.ts'; +import { getMaxMessageSize, getMaxChunkSize } from '../../../src/protocol/lish-protocol.ts'; import { downloadLimiter, uploadLimiter } from '../../../src/protocol/speed-limiter.ts'; -import { DEFAULT_MAX_MESSAGE_SIZE, type SettingsData } from '../../../src/settings.ts'; +import { DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_MAX_CHUNK_SIZE, type SettingsData } from '../../../src/settings.ts'; /** Minimal network settings slice — only the fields applyNetworkLimits reads. */ function netSlice(overrides: Partial): SettingsData['network'] { @@ -12,6 +12,7 @@ function netSlice(overrides: Partial): SettingsData['ne maxDownloadPeersPerLISH: 30, maxUploadPeersPerLISH: 30, maxMessageSize: DEFAULT_MAX_MESSAGE_SIZE, + maxChunkSize: DEFAULT_MAX_CHUNK_SIZE, ...overrides, } as SettingsData['network']; } @@ -23,10 +24,24 @@ describe('applyNetworkLimits', () => { }); it('pushes every limit from the settings snapshot into protocol module state', () => { - applyNetworkLimits(netSlice({ maxDownloadSpeed: 256, maxUploadSpeed: 128, maxMessageSize: 4 * 1024 * 1024 })); + applyNetworkLimits(netSlice({ maxDownloadSpeed: 256, maxUploadSpeed: 128, maxMessageSize: 4 * 1024 * 1024, maxChunkSize: 2 * 1024 * 1024 })); expect(downloadLimiter.getLimit()).toBe(256 * 1024); expect(uploadLimiter.getLimit()).toBe(128 * 1024); expect(getMaxMessageSize()).toBe(4 * 1024 * 1024); + expect(getMaxChunkSize()).toBe(2 * 1024 * 1024); + }); + + it('lifts a message limit that would be too small to carry one chunk', () => { + // A chunk is delivered as a single message: a message limit at or below the chunk + // limit would reject every chunk on arrival, so the chunk limit must win. + applyNetworkLimits(netSlice({ maxChunkSize: 8 * 1024 * 1024, maxMessageSize: 1024 })); + expect(getMaxChunkSize()).toBe(8 * 1024 * 1024); + expect(getMaxMessageSize()).toBeGreaterThan(8 * 1024 * 1024); + }); + + it('leaves a message limit that already clears the chunk limit alone', () => { + applyNetworkLimits(netSlice({ maxChunkSize: 2 * 1024 * 1024, maxMessageSize: 64 * 1024 * 1024 })); + expect(getMaxMessageSize()).toBe(64 * 1024 * 1024); }); it('is idempotent — re-applying the same snapshot keeps the same values', () => { diff --git a/backend/tests/unit/shared/expected-chunk-length.test.ts b/backend/tests/unit/shared/expected-chunk-length.test.ts new file mode 100644 index 00000000..e84c688c --- /dev/null +++ b/backend/tests/unit/shared/expected-chunk-length.test.ts @@ -0,0 +1,75 @@ +import { describe, it, expect } from 'bun:test'; +import { expectedChunkLength, type ILISH } from '@shared'; + +function makeLish(chunkSize: number, fileSizes: number[]): ILISH { + return { + id: 'lish-x', + created: new Date().toISOString(), + chunkSize, + checksumAlgo: 'sha256', + files: fileSizes.map((size, i) => ({ + path: `f${i}.bin`, + size, + checksums: new Array(size === 0 ? 0 : Math.ceil(size / chunkSize)).fill('h'), + })), + }; +} + +describe('expectedChunkLength', () => { + const CS = 1024; + + it('returns chunkSize for a full (non-last) chunk', () => { + const lish = makeLish(CS, [CS * 3]); + expect(expectedChunkLength(lish, 0, 0)).toBe(CS); + expect(expectedChunkLength(lish, 0, 1)).toBe(CS); + }); + + it('returns the shorter remainder for the last chunk of a file', () => { + const lish = makeLish(CS, [CS * 2 + 100]); // 3 chunks, last = 100 + expect(expectedChunkLength(lish, 0, 2)).toBe(100); + }); + + it('returns chunkSize for a last chunk that divides evenly', () => { + const lish = makeLish(CS, [CS * 2]); + expect(expectedChunkLength(lish, 0, 1)).toBe(CS); + }); + + it('handles a single sub-chunk-size file (one short chunk)', () => { + const lish = makeLish(CS, [200]); + expect(expectedChunkLength(lish, 0, 0)).toBe(200); + }); + + it('computes per-file independently across multiple files', () => { + const lish = makeLish(CS, [CS + 50, 300]); // file0: 2 chunks (last 50), file1: 1 chunk (300) + expect(expectedChunkLength(lish, 0, 0)).toBe(CS); + expect(expectedChunkLength(lish, 0, 1)).toBe(50); + expect(expectedChunkLength(lish, 1, 0)).toBe(300); + }); + + it('returns -1 for an out-of-range chunk index', () => { + const lish = makeLish(CS, [CS]); + expect(expectedChunkLength(lish, 0, 1)).toBe(-1); + expect(expectedChunkLength(lish, 0, -1)).toBe(-1); + }); + + it('returns -1 for an out-of-range or missing file', () => { + const lish = makeLish(CS, [CS]); + expect(expectedChunkLength(lish, 5, 0)).toBe(-1); + const { files: _files, ...noFiles } = lish; + expect(expectedChunkLength(noFiles, 0, 0)).toBe(-1); + }); + + it('returns -1 for an invalid chunkSize', () => { + const lish = makeLish(CS, [CS]); + expect(expectedChunkLength({ ...lish, chunkSize: 0 }, 0, 0)).toBe(-1); + }); + + // The download path rejects any chunk whose byte length != expectedChunkLength before + // hashing, so the trailing chunk must report the remainder rather than a full chunk. + it('reports the remainder for a trailing partial chunk', () => { + const lish = makeLish(CS, [CS * 2 + 100]); + expect(expectedChunkLength(lish, 0, 2)).toBe(100); + expect(expectedChunkLength(lish, 0, 0)).toBe(CS); + expect(expectedChunkLength(lish, 0, 1)).toBe(CS); + }); +}); diff --git a/backend/tests/unit/shared/validate-lish-structure.test.ts b/backend/tests/unit/shared/validate-lish-structure.test.ts index bacb1fef..4e2fd4c0 100644 --- a/backend/tests/unit/shared/validate-lish-structure.test.ts +++ b/backend/tests/unit/shared/validate-lish-structure.test.ts @@ -33,4 +33,67 @@ describe('validateLISHStructure', () => { it('rejects file with too many checksums', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024, files: [{ path: 'a.bin', size: 1024, checksums: ['h1', 'h2'] }] }), MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); // expected 1 it('rejects non-empty file with empty checksums', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024, files: [{ path: 'a.bin', size: 1, checksums: [] }] }), MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); it('rejects empty file with non-empty checksums', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024, files: [{ path: 'empty', size: 0, checksums: ['h1'] }] }), MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + + // Untrusted peer input: malformed shapes must yield a CodedError, never a native TypeError. + // toThrow() also guards the regression — a TypeError message would not contain the code. + it('rejects a null manifest', () => expect(() => validateLISHStructure(null as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a non-object manifest', () => expect(() => validateLISHStructure(42 as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects files that is not an array', () => expect(() => validateLISHStructure({ ...makeLish(), files: 5 } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a null file entry', () => expect(() => validateLISHStructure({ ...makeLish(), files: [null] } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + // Explicit size/chunkSize validation — the checksum-count equation alone lets these through. + it('rejects a negative file size (the ceil(-0) trick)', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024, files: [{ path: 'a.bin', size: -5, checksums: [] }] }), MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a string file size', () => expect(() => validateLISHStructure({ ...makeLish({ chunkSize: 1024 }), files: [{ path: 'a.bin', size: '1024', checksums: ['h1'] }] } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a float file size', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024, files: [{ path: 'a.bin', size: 1023.5, checksums: ['h1'] }] }), MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a float chunkSize', () => expect(() => validateLISHStructure(makeLish({ chunkSize: 1024.5 }), MAX)).toThrow(ErrorCodes.LISH_INVALID_CHUNK_SIZE)); + + // Presence vs truthiness: falsy non-array `files` is malformed, only absence means metadata-only. + it('rejects files: null', () => expect(() => validateLISHStructure({ ...makeLish(), files: null } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects files: false', () => expect(() => validateLISHStructure({ ...makeLish(), files: false } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + + // A checksum shared by slots with different expected lengths is unsatisfiable — one + // payload cannot be both full-length and shorter; the duplicate-slot write path would + // write past the shorter file tail. + it('rejects a checksum shared by a full chunk and a shorter last chunk', () => { + const lish = makeLish({ + chunkSize: 1024, + files: [ + { path: 'full.bin', size: 1024, checksums: ['dup'] }, + { path: 'short.bin', size: 1, checksums: ['dup'] }, + ], + }); + expect(() => validateLISHStructure(lish, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST); + }); + it('rejects a checksum shared by two last chunks of different lengths', () => { + const lish = makeLish({ + chunkSize: 1024, + files: [ + { path: 'a.bin', size: 1, checksums: ['dup'] }, + { path: 'b.bin', size: 2, checksums: ['dup'] }, + ], + }); + expect(() => validateLISHStructure(lish, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST); + }); + it('accepts duplicate checksums whose slots all expect the same length', () => { + const lish = makeLish({ + chunkSize: 1024, + files: [ + { path: 'a.bin', size: 1500, checksums: ['f', 's'] }, // full + short last + { path: 'copy.bin', size: 1500, checksums: ['f', 's'] }, // identical file — same lengths + { path: 'rep.bin', size: 2048, checksums: ['x', 'x'] }, // repeated full block within one file + ], + }); + expect(() => validateLISHStructure(lish, MAX)).not.toThrow(); + }); +}); + +describe('validateLISHStructure — optional arrays', () => { + it('rejects directories that is not an array', () => expect(() => validateLISHStructure({ ...makeLish(), directories: {} } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects links that is not an array', () => expect(() => validateLISHStructure({ ...makeLish(), links: {} } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('accepts absent directories and links', () => expect(() => validateLISHStructure(makeLish(), MAX)).not.toThrow()); +}); + +describe('validateLISHStructure — untrusted field types', () => { + it('rejects an unsupported checksumAlgo', () => expect(() => validateLISHStructure(makeLish({ checksumAlgo: 'md5' as never }), MAX)).toThrow(ErrorCodes.LISH_UNSUPPORTED_CHECKSUM)); + it('rejects a non-string file path', () => expect(() => validateLISHStructure({ ...makeLish({ chunkSize: 1024 }), files: [{ path: {}, size: 1024, checksums: ['h1'] }] } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); + it('rejects a non-string checksum entry', () => expect(() => validateLISHStructure({ ...makeLish({ chunkSize: 1024 }), files: [{ path: 'a.bin', size: 1024, checksums: [{}] }] } as unknown as ILISH, MAX)).toThrow(ErrorCodes.LISH_INVALID_MANIFEST)); }); diff --git a/frontend/src/pages/Network/Network.svelte b/frontend/src/pages/Network/Network.svelte index 87b49003..b7b7cedb 100644 --- a/frontend/src/pages/Network/Network.svelte +++ b/frontend/src/pages/Network/Network.svelte @@ -26,7 +26,7 @@ let activeTab = $state('lishs'); let tabDefs = $derived([ { id: 'lishs', icon: '/img/share.svg', label: $t('network.tabLishs') }, - { id: 'peers', icon: '/img/person.svg', label: $t('network.tabPeers') }, + { id: 'peers', icon: '/img/person.svg', label: $t('common.peers') }, ]); // =================== Peers data (shared between Peers tab and the LISH-peers popup) =================== diff --git a/frontend/src/pages/Network/NetworkLishs.svelte b/frontend/src/pages/Network/NetworkLishs.svelte index bf3b2b25..7641e1c8 100644 --- a/frontend/src/pages/Network/NetworkLishs.svelte +++ b/frontend/src/pages/Network/NetworkLishs.svelte @@ -158,7 +158,7 @@ {$t('network.lishID')} {$t('common.name')} {$t('network.totalSize')} - {$t('network.peerCount')} + {$t('common.peers')} {#each search.results as row, i (row.id)} diff --git a/frontend/src/pages/Settings/SettingsLISHNetworkList.svelte b/frontend/src/pages/Settings/SettingsLISHNetworkList.svelte index f1aea734..53cc23c1 100644 --- a/frontend/src/pages/Settings/SettingsLISHNetworkList.svelte +++ b/frontend/src/pages/Settings/SettingsLISHNetworkList.svelte @@ -21,7 +21,7 @@ import LISHNetworkExport from './SettingsLISHNetworkExport.svelte'; import LISHNetworkExportAll from './SettingsLISHNetworkExportAll.svelte'; import LISHNetworkPublic from './SettingsLISHNetworkPublic.svelte'; - import LISHNetworkBootstrap from './SettingsLISHNetworkBootstrap.svelte'; + import LISHNetworkBootstrap from './SettingsLISHNetworkPeers.svelte'; import NodeInfoRow from '../../components/NodeInfo/NodeInfoRow.svelte'; interface Props { areaID: string; @@ -61,7 +61,7 @@ function openBootstrap(network: LISHNetworkConfig): void { bootstrapNetwork = network; - bootstrapSubPage.enter(`${network.name} - ${$t('settings.lishNetwork.bootstrap.title')}`, () => void closeBootstrap()); + bootstrapSubPage.enter(`${network.name} - ${$t('common.peers')}`, () => void closeBootstrap()); } async function closeBootstrap(): Promise { bootstrapNetwork = null; diff --git a/frontend/src/pages/Settings/SettingsLISHNetworkBootstrap.svelte b/frontend/src/pages/Settings/SettingsLISHNetworkPeers.svelte similarity index 99% rename from frontend/src/pages/Settings/SettingsLISHNetworkBootstrap.svelte rename to frontend/src/pages/Settings/SettingsLISHNetworkPeers.svelte index b96abe78..cb98a5c5 100644 --- a/frontend/src/pages/Settings/SettingsLISHNetworkBootstrap.svelte +++ b/frontend/src/pages/Settings/SettingsLISHNetworkPeers.svelte @@ -21,7 +21,7 @@ import TableHeader from '../../components/Table/TableHeader.svelte'; import TableRow from '../../components/Table/TableRow.svelte'; import TableCell from '../../components/Table/TableCell.svelte'; - import LISHNetworkBootstrapPeer from './SettingsLISHNetworkBootstrapPeer.svelte'; + import LISHNetworkBootstrapPeer from './SettingsLISHNetworkPeersPeer.svelte'; interface Props { areaID: string; position?: Position | undefined; diff --git a/frontend/src/pages/Settings/SettingsLISHNetworkBootstrapPeer.svelte b/frontend/src/pages/Settings/SettingsLISHNetworkPeersPeer.svelte similarity index 100% rename from frontend/src/pages/Settings/SettingsLISHNetworkBootstrapPeer.svelte rename to frontend/src/pages/Settings/SettingsLISHNetworkPeersPeer.svelte diff --git a/frontend/src/scripts/peerFallback.ts b/frontend/src/scripts/peerFallback.ts index e8736376..428d6712 100644 --- a/frontend/src/scripts/peerFallback.ts +++ b/frontend/src/scripts/peerFallback.ts @@ -23,11 +23,16 @@ export const FALLBACK_DEADLINE_MS = 5 * 60 * 1000; * (`PEER_*` code — unreachable, stopped sharing, busy, I/O trouble) or an error the * caller explicitly flagged `tryNextPeer`. Local errors (e.g. LISH already added) * are not retryable — they would fail identically on every peer. + * + * `LISH_CHUNK_SIZE_TOO_LARGE` counts as local: the chunk size is written in the LISH + * itself, so every honest peer answers with the same value. Walking the whole peer list + * only makes the user watch each row fail before the identical error appears at the end. */ export function isRetryablePeerError(error: unknown): boolean { if ((error as { tryNextPeer?: boolean } | null)?.tryNextPeer) return true; const code = (error as { code?: unknown } | null)?.code; - return typeof code === 'string' && code.startsWith('PEER_'); + if (typeof code !== 'string') return false; + return code.startsWith('PEER_'); } /** diff --git a/frontend/src/scripts/settings.ts b/frontend/src/scripts/settings.ts index a144d6dd..5b5f8278 100644 --- a/frontend/src/scripts/settings.ts +++ b/frontend/src/scripts/settings.ts @@ -1,4 +1,5 @@ import { get, writable, type Writable } from 'svelte/store'; +import { minMessageSizeFor } from '@shared'; import { api } from './api.ts'; import { defaultWidgetVisibility, type FooterPosition, type FooterWidget } from './footerWidgets.ts'; import { currentLanguage, languages } from './language.ts'; @@ -246,10 +247,16 @@ export function setMaxUploadSpeed(value: number): void { export function setMaxChunkSize(value: number): void { const clampedValue = Math.max(1, value || 1); updateSetting(maxChunkSize, 'network.maxChunkSize', clampedValue); + // Raising the chunk limit above the message limit would break every transfer, so pull + // the message limit up with it instead of leaving an unusable pair on screen. The backend + // enforces the same floor — doing it here too spares the UI a round-trip. + const floor = minMessageSizeFor(clampedValue); + if (get(maxMessageSize) < floor) updateSetting(maxMessageSize, 'network.maxMessageSize', floor); } export function setMaxMessageSize(value: number): void { - const clampedValue = Math.max(1, value || 1); + const floor = minMessageSizeFor(get(maxChunkSize)); + const clampedValue = Math.max(floor, value || 1); updateSetting(maxMessageSize, 'network.maxMessageSize', clampedValue); } diff --git a/frontend/static/langs/cs.json b/frontend/static/langs/cs.json index 9e21beac..df23e3c0 100644 --- a/frontend/static/langs/cs.json +++ b/frontend/static/langs/cs.json @@ -32,6 +32,7 @@ "uploads": "Odesílání", "files": "Soubory", "connections": "Spojení", + "peers": "Účastníci", "speed": "Rychlost", "transferred": "Přeneseno", "newDirectory": "Nový adresář", @@ -85,7 +86,6 @@ "network": { "title": "Procházet síť", "tabLishs": "LISHe", - "tabPeers": "Účastníci", "searchPlaceholder": "Hledat podle ID účastníka ...", "searchLishsPlaceholder": "Hledat LISH podle názvu nebo ID ...", "searching": "Prohledávám síť ...", @@ -98,7 +98,6 @@ "openPeer": "Otevřít účastníka", "allNetworks": "Všechny sítě", "peerID": "ID účastníka", - "peerCount": "Účastníků", "network": "Síť", "connections": "Připojení", "direct": "{count} přímé", @@ -278,7 +277,7 @@ "errorMissingCreated": "Neplatný LISH: chybí nebo prázdný časový údaj vytvoření", "errorInvalidChunkSize": "Neplatný LISH: chybí nebo neplatná velikost části", "errorChunkSizeTooLarge": "LISH odmítnut: velikost části přesahuje povolený limit ({detail})", - "errorInvalidManifest": "Neplatný LISH: počet kontrolních součtů neodpovídá deklarované velikosti souboru ({detail})", + "errorInvalidManifest": "Neplatný manifest LISHe: {detail}", "errorUnsupportedChecksum": "Neplatný LISH: nepodporovaný hashovací algoritmus: {detail}", "errorUnexpectedArray": "Očekáván jeden LISH objekt, obdrženo pole", "errorPathAccessDenied": "Nelze přistoupit k cestě: {detail}", @@ -377,7 +376,6 @@ "stable": "Síť je stabilní - broadcasty dorazí k fleetu" }, "bootstrap": { - "title": "Bootstrap účastníci", "discoveredShort": "z gossipu", "discoveredHelp": "Získáni za běhu přes gossip. Automaticky odstraněni při neplatné identitě.", "originConfigured": "uložený", diff --git a/frontend/static/langs/en.json b/frontend/static/langs/en.json index 4a7a1cd5..d3a78b38 100644 --- a/frontend/static/langs/en.json +++ b/frontend/static/langs/en.json @@ -32,6 +32,7 @@ "uploads": "Uploads", "files": "Files", "connections": "Connections", + "peers": "Peers", "speed": "Speed", "transferred": "Transferred", "newDirectory": "New directory", @@ -85,7 +86,6 @@ "network": { "title": "Browse network", "tabLishs": "LISHs", - "tabPeers": "Peers", "searchPlaceholder": "Search by peer ID ...", "searchLishsPlaceholder": "Search LISHs by name or ID ...", "searching": "Searching network ...", @@ -98,7 +98,6 @@ "openPeer": "Open peer", "allNetworks": "All networks", "peerID": "Peer ID", - "peerCount": "Peers", "network": "Network", "connections": "Connections", "direct": "{count} direct", @@ -278,7 +277,7 @@ "errorMissingCreated": "Invalid LISH: missing or empty created timestamp", "errorInvalidChunkSize": "Invalid LISH: missing or invalid chunk size", "errorChunkSizeTooLarge": "LISH rejected: chunk size exceeds the configured limit ({detail})", - "errorInvalidManifest": "Invalid LISH: number of checksums does not match declared file size ({detail})", + "errorInvalidManifest": "Invalid LISH manifest: {detail}", "errorUnsupportedChecksum": "Invalid LISH: unsupported checksum algorithm: {detail}", "errorUnexpectedArray": "Expected a single LISH object, got an array", "errorPathAccessDenied": "Cannot access path: {detail}", @@ -377,7 +376,6 @@ "stable": "Mesh is stable — broadcasts will reach the fleet" }, "bootstrap": { - "title": "Bootstrap peers", "discoveredShort": "gossip", "discoveredHelp": "Learned at runtime from gossip. Auto-removed when stale.", "originConfigured": "saved", diff --git a/frontend/tests/unit/peer-fallback.test.ts b/frontend/tests/unit/peer-fallback.test.ts index e85a5788..e0ea29bf 100644 --- a/frontend/tests/unit/peer-fallback.test.ts +++ b/frontend/tests/unit/peer-fallback.test.ts @@ -25,6 +25,9 @@ test('isRetryablePeerError classifies peer-side, flagged and local errors', () = expect(isRetryablePeerError(codedError('PEER_LISH_NOT_SHARED'))).toBe(true); expect(isRetryablePeerError(codedError('PEER_BUSY'))).toBe(true); expect(isRetryablePeerError(Object.assign(new Error('declined'), { tryNextPeer: true }))).toBe(true); + // Chunk size is a property of the LISH, identical from every honest peer — asking + // the rest would only repeat the same answer, so the error stops the loop at once. + expect(isRetryablePeerError(codedError('LISH_CHUNK_SIZE_TOO_LARGE'))).toBe(false); expect(isRetryablePeerError(codedError('LISH_ALREADY_EXISTS'))).toBe(false); expect(isRetryablePeerError(new Error('plain'))).toBe(false); expect(isRetryablePeerError(null)).toBe(false); @@ -146,3 +149,28 @@ test('stops starting new attempts once the deadline has passed', async () => { // Deadline already expired — only the first peer gets an attempt. expect(tried).toEqual([PEERS[0]!.peerID]); }); + +test('an over-limit LISH stops at the first peer and leaves the rest unmarked', async () => { + // Reproduces the reported case: a search result offered by five peers where the LISH + // declares a chunk size above the local limit. Only the first peer may be asked, and no + // row may be branded unavailable — the peers are fine, the LISH is simply too coarse. + const fivePeers: PeerRef[] = Array.from({ length: 5 }, (_, i) => ({ peerID: `peer-${i}`, networkID: 'net-1' })); + const asked: string[] = []; + const statuses: Array<[number, PeerAttemptStatus | null]> = []; + + const err = await withPeerFallback( + fivePeers, + async peerID => { + asked.push(peerID); + throw codedError('LISH_CHUNK_SIZE_TOO_LARGE'); + }, + (i, s) => statuses.push([i, s]) + ).catch(e => e); + + expect((err as { code?: string }).code).toBe('LISH_CHUNK_SIZE_TOO_LARGE'); + expect(asked).toEqual(['peer-0']); + expect(statuses).toEqual([ + [0, 'downloading'], + [0, null], + ]); +}); diff --git a/shared/src/lish.ts b/shared/src/lish.ts index dd38d2de..f26aa7a0 100644 --- a/shared/src/lish.ts +++ b/shared/src/lish.ts @@ -1,4 +1,5 @@ import { CodedError, ErrorCodes } from './errors.ts'; +import { formatBytes } from './utils.ts'; export type LISHid = string; export type ChunkID = string; export const SUPPORTED_ALGOS = ['sha256', 'sha384', 'sha512', 'sha512-256', 'sha3-256', 'sha3-384', 'sha3-512', 'blake2b256', 'blake2b512', 'blake2s256'] as const; @@ -6,6 +7,28 @@ export type HashAlgorithm = (typeof SUPPORTED_ALGOS)[number]; export const DEFAULT_ALGO: HashAlgorithm = 'sha256'; export const DEFAULT_CHUNK_SIZE: number = 1024 * 1024; +/** + * A chunk travels as one message together with its envelope (type tag, LISH id, chunk id, + * msgpack framing), so the message-size limit must stay this far above the chunk-size limit. + */ +export const MESSAGE_SIZE_HEADROOM: number = 1024 * 1024; + +/** Smallest message limit that can still carry a chunk of the given size. */ +export function minMessageSizeFor(maxChunkSize: number): number { + return maxChunkSize + MESSAGE_SIZE_HEADROOM; +} + +/** + * Render an "actual > limit" size pair for an error detail. formatBytes rounds, so a value + * one byte over the limit would print as "1 MB > 1 MB"; when both sides collapse to the same + * text, fall back to raw bytes so the message still tells the user something. + */ +export function formatSizeOverLimit(actual: number, limit: number): string { + const a = formatBytes(actual); + const l = formatBytes(limit); + return a === l ? `${actual} B > ${limit} B` : `${a} > ${l}`; +} + /** * Validate chunkSize bounds and manifest consistency. * Throws CodedError on the first violation. @@ -16,19 +39,85 @@ export const DEFAULT_CHUNK_SIZE: number = 1024 * 1024; * (zero-size files have zero checksums) */ export function validateLISHStructure(lish: ILISH, maxChunkSize: number): void { - if (typeof lish.chunkSize !== 'number' || !Number.isFinite(lish.chunkSize) || lish.chunkSize <= 0) throw new CodedError(ErrorCodes.LISH_INVALID_CHUNK_SIZE, String(lish.chunkSize)); - if (lish.chunkSize > maxChunkSize) throw new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, `${lish.chunkSize} > ${maxChunkSize}`); - if (lish.files) { + // `lish` may come straight off the wire from an untrusted peer — reject malformed shapes with + // a coded error rather than letting a raw property access throw a native TypeError. + if (!lish || typeof lish !== 'object') throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, 'manifest is not an object'); + if (typeof lish.chunkSize !== 'number' || !Number.isInteger(lish.chunkSize) || lish.chunkSize <= 0) throw new CodedError(ErrorCodes.LISH_INVALID_CHUNK_SIZE, String(lish.chunkSize)); + if (lish.chunkSize > maxChunkSize) throw new CodedError(ErrorCodes.LISH_CHUNK_SIZE_TOO_LARGE, formatSizeOverLimit(lish.chunkSize, maxChunkSize)); + // An unsupported checksumAlgo would later crash `new Bun.CryptoHasher(algo)` during + // download/verify — reject the peer manifest here instead, matching validateImportedLISH. + if (typeof lish.checksumAlgo !== 'string' || !(SUPPORTED_ALGOS as readonly string[]).includes(lish.checksumAlgo)) throw new CodedError(ErrorCodes.LISH_UNSUPPORTED_CHECKSUM, String(lish.checksumAlgo)); + // Optional arrays get the same presence check as `files`: a truthy non-array + // (e.g. `directories: {}`) would crash downstream for..of iteration in + // dataServer.add with a raw TypeError instead of a coded rejection. + if (lish.directories !== undefined && !Array.isArray(lish.directories)) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, 'directories is not an array'); + if (lish.links !== undefined && !Array.isArray(lish.links)) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, 'links is not an array'); + // Presence check, not truthiness: `files: null` (or any other falsy non-array) is a + // malformed manifest, only a genuinely absent field means metadata-only. + if (lish.files !== undefined) { + if (!Array.isArray(lish.files)) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, 'files is not an array'); for (const file of lish.files) { + if (!file || typeof file !== 'object') throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, 'file entry is not an object'); + // path must be a string — a non-string (e.g. {}) would blow up the SQLite bind + // in dataServer.add with a raw error instead of a coded peer rejection. + if (typeof file.path !== 'string') throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `file path is not a string: ${String(file.path)}`); + // Explicit size validation — the checksum-count equation alone lets adversarial + // sizes through (e.g. size -5 with 0 checksums: ceil(-5/cs) is -0 and 0 !== -0 is + // false) and a float size makes every chunk "wrong length", banning honest peers. + if (typeof file.size !== 'number' || !Number.isInteger(file.size) || file.size < 0) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `${file.path}: invalid size ${String(file.size)}`); const expected = file.size === 0 ? 0 : Math.ceil(file.size / lish.chunkSize); if (!Array.isArray(file.checksums) || file.checksums.length !== expected) { const got = Array.isArray(file.checksums) ? file.checksums.length : 'invalid'; throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `${file.path}: expected ${expected} checksums for size ${file.size} / chunkSize ${lish.chunkSize}, got ${got}`); } + // Each checksum must be a string too — the download path compares it to a hex + // digest and the DB binds it; a non-string entry would corrupt both. + for (const cs of file.checksums) if (typeof cs !== 'string') throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `${file.path}: non-string checksum`); + } + // A checksum names exact content, so every slot sharing it must expect the same byte + // length — otherwise one verified payload cannot satisfy all its slots and the + // duplicate-slot write path would write a full chunk past a shorter file tail. Only a + // file's short last chunk can differ from chunkSize, so tracking those keeps this + // O(#files) in memory. + const shortLast = new Map(); + for (const file of lish.files) { + const rem = file.size % lish.chunkSize; + if (file.size > 0 && rem !== 0) { + const cs = file.checksums[file.checksums.length - 1]!; + const prev = shortLast.get(cs); + if (prev !== undefined && prev !== rem) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `${file.path}: duplicate checksum with conflicting chunk lengths (${prev} vs ${rem})`); + shortLast.set(cs, rem); + } + } + if (shortLast.size > 0) { + for (const file of lish.files) { + const rem = file.size % lish.chunkSize; + const shortLastIdx = file.size > 0 && rem !== 0 ? file.checksums.length - 1 : -1; + for (let i = 0; i < file.checksums.length; i++) { + if (i === shortLastIdx) continue; // consistency of short last chunks verified above + const shortLen = shortLast.get(file.checksums[i]!); + if (shortLen !== undefined) throw new CodedError(ErrorCodes.LISH_INVALID_MANIFEST, `${file.path}: duplicate checksum with conflicting chunk lengths (${shortLen} vs ${lish.chunkSize})`); + } + } } } } +/** + * Exact byte length of a single chunk as fixed by the manifest. + * The last chunk of a file may be shorter than `chunkSize`. Returns -1 when the length + * cannot be determined (no file metadata, invalid chunkSize/size, or out-of-range indices). + */ +export function expectedChunkLength(lish: ILISH, fileIndex: number, chunkIndex: number): number { + if (!lish.files || fileIndex < 0 || fileIndex >= lish.files.length) return -1; + if (typeof lish.chunkSize !== 'number' || !Number.isFinite(lish.chunkSize) || lish.chunkSize <= 0) return -1; + const size = lish.files[fileIndex]!.size; + if (typeof size !== 'number' || !Number.isFinite(size) || size < 0) return -1; + const numChunks = size === 0 ? 0 : Math.ceil(size / lish.chunkSize); + if (chunkIndex < 0 || chunkIndex >= numChunks) return -1; + return Math.min(lish.chunkSize, size - chunkIndex * lish.chunkSize); +} + export interface ILISH { id: string; name?: string | undefined;