diff --git a/packages/opencode/src/config/config.ts b/packages/opencode/src/config/config.ts index 0ede11844e4..0765284452b 100644 --- a/packages/opencode/src/config/config.ts +++ b/packages/opencode/src/config/config.ts @@ -1193,6 +1193,16 @@ export namespace Config { url: z.string().optional().describe("Enterprise URL"), }) .optional(), + retention: z + .object({ + days: z + .number() + .int() + .min(0) + .optional() + .describe("Auto-delete archived sessions older than this many days (default: 90, 0 = disabled)"), + }) + .optional(), compaction: z .object({ auto: z.boolean().optional().describe("Enable automatic compaction when context is full (default: true)"), diff --git a/packages/opencode/src/lsp/client.ts b/packages/opencode/src/lsp/client.ts index de0c4386268..27e749f6641 100644 --- a/packages/opencode/src/lsp/client.ts +++ b/packages/opencode/src/lsp/client.ts @@ -57,7 +57,12 @@ export namespace LSPClient { count: params.diagnostics.length, }) const exists = diagnostics.has(filePath) - diagnostics.set(filePath, params.diagnostics) + if (params.diagnostics.length === 0) { + if (!exists) return + diagnostics.delete(filePath) + } else { + diagnostics.set(filePath, params.diagnostics) + } if (!exists && input.serverID === "typescript") return Bus.publish(Event.Diagnostics, { path: filePath, serverID: input.serverID }) }) diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index f6145b7a47e..33244f358cf 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -97,6 +97,9 @@ export namespace SessionCompaction { for (const part of toPrune) { if (part.state.status === "completed") { part.state.time.compacted = Date.now() + part.state.output = "[compacted]" + part.state.metadata = {} + part.state.attachments = undefined await Session.updatePart(part) } } diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index b3c34539e77..1748d2b353c 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -1,8 +1,10 @@ import path from "path" import os from "os" import fs from "fs/promises" +import fsSync from "fs" import z from "zod" import { Filesystem } from "../util/filesystem" +import { Identifier } from "@/id/id" import { SessionID, MessageID, PartID } from "./schema" import { MessageV2 } from "./message-v2" import { Log } from "../util/log" @@ -1713,29 +1715,37 @@ NOTE: At any point in time through this workflow you should feel free to ask the }, }) - let output = "" + let head = "" + let spoolFd: number | undefined + let spoolPath: string | undefined + let totalBytes = 0 - proc.stdout?.on("data", (chunk) => { - output += chunk.toString() - if (part.state.status === "running") { - part.state.metadata = { - output: output, - description: "", + const appendShell = (chunk: Buffer) => { + const str = chunk.toString() + totalBytes += str.length + if (head.length < Truncate.MAX_BYTES) { + head += str + } + if (totalBytes > Truncate.MAX_BYTES) { + if (spoolFd === undefined) { + spoolPath = path.join(Truncate.DIR, Identifier.ascending("tool")) + fsSync.mkdirSync(Truncate.DIR, { recursive: true }) + spoolFd = fsSync.openSync(spoolPath, "w") + fsSync.writeSync(spoolFd, head) } - Session.updatePart(part) + fsSync.writeSync(spoolFd, str) } - }) - - proc.stderr?.on("data", (chunk) => { - output += chunk.toString() if (part.state.status === "running") { part.state.metadata = { - output: output, + output: head, description: "", } Session.updatePart(part) } - }) + } + + proc.stdout?.on("data", appendShell) + proc.stderr?.on("data", appendShell) let aborted = false let exited = false @@ -1762,9 +1772,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the }) }) + if (spoolFd !== undefined) fsSync.closeSync(spoolFd) + if (aborted) { - output += "\n\n" + ["", "User aborted the command", ""].join("\n") + head += "\n\n" + ["", "User aborted the command", ""].join("\n") } + + // Truncate output for storage — full output already on disk via spool + const truncated = await Truncate.output(head, {}) + const stored = truncated.truncated ? truncated.content : head + msg.time.completed = Date.now() await Session.updateMessage(msg) if (part.state.status === "running") { @@ -1777,10 +1794,13 @@ NOTE: At any point in time through this workflow you should feel free to ask the input: part.state.input, title: "", metadata: { - output, + output: stored, description: "", + truncated: truncated.truncated, + ...(truncated.truncated && { outputPath: truncated.outputPath }), + ...(spoolPath && !truncated.truncated && { outputPath: spoolPath }), }, - output, + output: stored, } await Session.updatePart(part) } diff --git a/packages/opencode/src/storage/db.ts b/packages/opencode/src/storage/db.ts index f41a1ecd854..1ee99056d86 100644 --- a/packages/opencode/src/storage/db.ts +++ b/packages/opencode/src/storage/db.ts @@ -94,6 +94,15 @@ export namespace Database { db.run("PRAGMA foreign_keys = ON") db.run("PRAGMA wal_checkpoint(PASSIVE)") + // Migrate to incremental auto-vacuum if needed (one-time) + const vacuum = db.$client.prepare("PRAGMA auto_vacuum").get() as { auto_vacuum: number } | undefined + if (vacuum && vacuum.auto_vacuum === 0) { + log.info("migrating database to incremental auto-vacuum mode (one-time operation)...") + db.$client.run("PRAGMA auto_vacuum = 2") + db.$client.run("VACUUM") + log.info("auto-vacuum migration complete") + } + // Apply schema migrations const entries = typeof OPENCODE_MIGRATIONS !== "undefined" @@ -115,6 +124,16 @@ export namespace Database { return db }) + /** Run periodic WAL checkpoint (TRUNCATE mode) */ + export function checkpoint() { + Client().$client.run("PRAGMA wal_checkpoint(TRUNCATE)") + } + + /** Run incremental vacuum to reclaim free pages */ + export function vacuum() { + Client().$client.run("PRAGMA incremental_vacuum(500)") + } + export function close() { Client().$client.close() Client.reset() diff --git a/packages/opencode/src/tool/bash.ts b/packages/opencode/src/tool/bash.ts index 50ae4abac8d..01dcfb39f74 100644 --- a/packages/opencode/src/tool/bash.ts +++ b/packages/opencode/src/tool/bash.ts @@ -2,12 +2,12 @@ import z from "zod" import { spawn } from "child_process" import { Tool } from "./tool" import path from "path" +import fs from "fs" import DESCRIPTION from "./bash.txt" import { Log } from "../util/log" import { Instance } from "../project/instance" import { lazy } from "@/util/lazy" import { Language } from "web-tree-sitter" -import fs from "fs/promises" import { Filesystem } from "@/util/filesystem" import { fileURLToPath } from "url" @@ -16,6 +16,7 @@ import { Shell } from "@/shell/shell" import { BashArity } from "@/permission/arity" import { Truncate } from "./truncate" +import { Identifier } from "@/id/id" import { Plugin } from "@/plugin" const MAX_METADATA_LENGTH = 30_000 @@ -116,7 +117,7 @@ export const BashTool = Tool.define("bash", async () => { if (["cd", "rm", "cp", "mv", "mkdir", "touch", "chmod", "chown", "cat"].includes(command[0])) { for (const arg of command.slice(1)) { if (arg.startsWith("-") || (command[0] === "chmod" && arg.startsWith("+"))) continue - const resolved = await fs.realpath(path.resolve(cwd, arg)).catch(() => "") + const resolved = await fs.promises.realpath(path.resolve(cwd, arg)).catch(() => "") log.info("resolved path", { arg, resolved }) if (resolved) { const normalized = @@ -176,7 +177,10 @@ export const BashTool = Tool.define("bash", async () => { windowsHide: process.platform === "win32", }) - let output = "" + let head = "" + let spoolFd: number | undefined + let spoolPath: string | undefined + let totalBytes = 0 // Initialize metadata with empty output ctx.metadata({ @@ -187,11 +191,28 @@ export const BashTool = Tool.define("bash", async () => { }) const append = (chunk: Buffer) => { - output += chunk.toString() + const str = chunk.toString() + totalBytes += str.length + // Keep only enough in memory for the metadata preview and final + // truncated output (Truncate.MAX_BYTES = 50 KB). Everything else + // goes straight to a spool file on disk so the full output is + // recoverable via grep/read without blowing up the heap. + if (head.length < Truncate.MAX_BYTES) { + head += str + } + if (totalBytes > Truncate.MAX_BYTES) { + if (spoolFd === undefined) { + spoolPath = path.join(Truncate.DIR, Identifier.ascending("tool")) + fs.mkdirSync(Truncate.DIR, { recursive: true }) + spoolFd = fs.openSync(spoolPath, "w") + // Flush everything accumulated so far + fs.writeSync(spoolFd, head) + } + fs.writeSync(spoolFd, str) + } ctx.metadata({ metadata: { - // truncate the metadata to avoid GIANT blobs of data (has nothing to do w/ what agent can access) - output: output.length > MAX_METADATA_LENGTH ? output.slice(0, MAX_METADATA_LENGTH) + "\n\n..." : output, + output: head.length > MAX_METADATA_LENGTH ? head.slice(0, MAX_METADATA_LENGTH) + "\n\n..." : head, description: params.description, }, }) @@ -242,28 +263,44 @@ export const BashTool = Tool.define("bash", async () => { }) }) - const resultMetadata: string[] = [] + if (spoolFd !== undefined) fs.closeSync(spoolFd) + + const suffix: string[] = [] if (timedOut) { - resultMetadata.push(`bash tool terminated command after exceeding timeout ${timeout} ms`) + suffix.push(`bash tool terminated command after exceeding timeout ${timeout} ms`) } if (aborted) { - resultMetadata.push("User aborted the command") + suffix.push("User aborted the command") + } + + if (suffix.length > 0) { + head += "\n\n\n" + suffix.join("\n") + "\n" } - if (resultMetadata.length > 0) { - output += "\n\n\n" + resultMetadata.join("\n") + "\n" + const meta = { + output: head.length > MAX_METADATA_LENGTH ? head.slice(0, MAX_METADATA_LENGTH) + "\n\n..." : head, + exit: proc.exitCode, + description: params.description, + } + + // If output was spooled to disk, run truncation ourselves and signal + // the Tool.define wrapper to skip its own Truncate.output() pass. + if (spoolPath) { + const preview = head.slice(0, Truncate.MAX_BYTES) + const hint = `The tool call succeeded but the output was truncated. Full output saved to: ${spoolPath}\nUse Grep to search the full content or Read with offset/limit to view specific sections.` + return { + title: params.description, + metadata: { ...meta, truncated: true as const, outputPath: spoolPath }, + output: `${preview}\n\n...${totalBytes - preview.length} bytes truncated...\n\n${hint}`, + } } return { title: params.description, - metadata: { - output: output.length > MAX_METADATA_LENGTH ? output.slice(0, MAX_METADATA_LENGTH) + "\n\n..." : output, - exit: proc.exitCode, - description: params.description, - }, - output, + metadata: meta, + output: head, } }, } diff --git a/packages/opencode/src/tool/edit.ts b/packages/opencode/src/tool/edit.ts index 554d547d051..ab8df53c393 100644 --- a/packages/opencode/src/tool/edit.ts +++ b/packages/opencode/src/tool/edit.ts @@ -173,24 +173,21 @@ const SINGLE_CANDIDATE_SIMILARITY_THRESHOLD = 0.0 const MULTIPLE_CANDIDATES_SIMILARITY_THRESHOLD = 0.3 /** - * Levenshtein distance algorithm implementation + * Levenshtein distance — 2-row O(min(n,m)) memory instead of full O(n×m) matrix. */ function levenshtein(a: string, b: string): number { - // Handle empty strings - if (a === "" || b === "") { - return Math.max(a.length, b.length) - } - const matrix = Array.from({ length: a.length + 1 }, (_, i) => - Array.from({ length: b.length + 1 }, (_, j) => (i === 0 ? j : j === 0 ? i : 0)), - ) - + if (a === "" || b === "") return Math.max(a.length, b.length) + if (a.length < b.length) [a, b] = [b, a] + let prev = Array.from({ length: b.length + 1 }, (_, j) => j) + let curr = new Array(b.length + 1) for (let i = 1; i <= a.length; i++) { + curr[0] = i for (let j = 1; j <= b.length; j++) { - const cost = a[i - 1] === b[j - 1] ? 0 : 1 - matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + cost) + curr[j] = a[i - 1] === b[j - 1] ? prev[j - 1] : 1 + Math.min(prev[j], curr[j - 1], prev[j - 1]) } + ;[prev, curr] = [curr, prev] } - return matrix[a.length][b.length] + return prev[b.length] } export const SimpleReplacer: Replacer = function* (_content, find) { diff --git a/packages/opencode/src/util/rpc.ts b/packages/opencode/src/util/rpc.ts index ebd8be40e45..9f3279730dc 100644 --- a/packages/opencode/src/util/rpc.ts +++ b/packages/opencode/src/util/rpc.ts @@ -44,10 +44,17 @@ export namespace Rpc { } return { call(method: Method, input: Parameters[0]): Promise> { - const requestId = id++ - return new Promise((resolve) => { - pending.set(requestId, resolve) - target.postMessage(JSON.stringify({ type: "rpc.request", method, input, id: requestId })) + const rid = id++ + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pending.delete(rid) + reject(new Error(`RPC timeout: ${String(method)}`)) + }, 60_000) + pending.set(rid, (result) => { + clearTimeout(timer) + resolve(result) + }) + target.postMessage(JSON.stringify({ type: "rpc.request", method, input, id: rid })) }) }, on(event: string, handler: (data: Data) => void) { diff --git a/packages/opencode/test/tool/bash.test.ts b/packages/opencode/test/tool/bash.test.ts index 4d680d494f3..90c43cf2eb1 100644 --- a/packages/opencode/test/tool/bash.test.ts +++ b/packages/opencode/test/tool/bash.test.ts @@ -400,4 +400,36 @@ describe("tool.bash truncation", () => { }, }) }) + + test("spools large output to disk and keeps full content recoverable", async () => { + await Instance.provide({ + directory: projectRoot, + fn: async () => { + const bash = await BashTool.init() + // Generate 200 KB of output — well over the 50 KB spool threshold + const bytes = Truncate.MAX_BYTES * 4 + const result = await bash.execute( + { + command: `head -c ${bytes} /dev/urandom | base64`, + description: "Generate large output for spool test", + }, + ctx, + ) + expect((result.metadata as any).truncated).toBe(true) + + const filepath = (result.metadata as any).outputPath + expect(filepath).toBeTruthy() + // Spool file should be in the tool-output directory + expect(filepath).toContain("tool-output") + + // Full output on disk should be larger than what we kept in memory + const saved = await Filesystem.readText(filepath) + expect(saved.length).toBeGreaterThan(Truncate.MAX_BYTES) + + // The returned output to the agent should be truncated + expect(result.output.length).toBeLessThan(Truncate.MAX_BYTES * 2) + expect(result.output).toContain("truncated") + }, + }) + }) })