diff --git a/plugins/chassis/src/image-mime.ts b/plugins/chassis/src/image-mime.ts new file mode 100644 index 0000000000..b345890ee4 --- /dev/null +++ b/plugins/chassis/src/image-mime.ts @@ -0,0 +1,18 @@ +const RENDERABLE_IMAGE_TYPES = new Set([ + "image/png", + "image/jpeg", + "image/jpg", + "image/pjpeg", + "image/gif", + "image/webp", + "image/avif", + "image/bmp", + "image/apng", + "image/x-icon", + "image/vnd.microsoft.icon", + "image/x-ms-bmp", +]); + +export function browserRenderableImage(mimeType?: string): boolean { + return RENDERABLE_IMAGE_TYPES.has((mimeType ?? "").split(";")[0]!.trim().toLowerCase()); +} diff --git a/plugins/web-ui/server/index.ts b/plugins/web-ui/server/index.ts index 9fb679df17..4fb8c2b685 100644 --- a/plugins/web-ui/server/index.ts +++ b/plugins/web-ui/server/index.ts @@ -789,6 +789,8 @@ interface CoreAttachment { mimetype: string; sizeBytes: number; blobId: string; + previewBlobId?: string; + previewMimetype?: string; } interface CoreApprovalRecord { @@ -1038,7 +1040,7 @@ async function serveFileContent(c: WebCtx, playground = false): Promise const { res, user, url } = c; const id = c.params.id!; const corePath = withSourceAuthNonce( - `/v1/files/${encodeURIComponent(id)}/content?viewer=${encodeURIComponent(user)}`, + `/v1/files/${encodeURIComponent(id)}/content?viewer=${encodeURIComponent(user)}${url.searchParams.get("preview") === "1" ? "&preview=1" : ""}`, CORE_SIGNING_SECRET, ); const portalTok = portalTokenStore.getStore(); @@ -2330,13 +2332,27 @@ const apiRoutes: readonly WebRoute[] = [ if (Array.isArray(p.attachments)) { for (const raw of p.attachments as unknown[]) { if (!raw || typeof raw !== "object") continue; - const a = raw as { name?: unknown; mimetype?: unknown; sizeBytes?: unknown; blobId?: unknown }; + const a = raw as { + name?: unknown; + mimetype?: unknown; + sizeBytes?: unknown; + blobId?: unknown; + previewBlobId?: unknown; + previewMimetype?: unknown; + }; if (typeof a.name !== "string" || typeof a.blobId !== "string" || !a.blobId) continue; attachments.push({ name: a.name, mimetype: typeof a.mimetype === "string" && a.mimetype ? a.mimetype : "application/octet-stream", sizeBytes: typeof a.sizeBytes === "number" ? a.sizeBytes : 0, blobId: a.blobId, + ...(typeof a.previewBlobId === "string" && a.previewBlobId + ? { + previewBlobId: a.previewBlobId, + previewMimetype: + typeof a.previewMimetype === "string" ? a.previewMimetype : "application/octet-stream", + } + : {}), }); } } diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.ts index b80ff16b84..367e74f03f 100644 --- a/plugins/web-ui/src/chat.ts +++ b/plugins/web-ui/src/chat.ts @@ -686,6 +686,8 @@ export function createChatSurface( function dispose(): void { redrawHooks.delete(redrawForConnector); + for (const url of localAttachmentUrls.values()) URL.revokeObjectURL(url); + localAttachmentUrls.clear(); teardownActiveChat(); } @@ -1630,7 +1632,13 @@ export function createChatSurface( ${edited || deleted ? html`(${deleted ? "deleted" : "edited"})` : nothing} - ${attachments.length ? html`
${attachments.map(userAttachmentBadge)}
` : nothing} + ${ + attachments.length + ? html`
+ ${attachments.map((attachment) => userAttachmentBadge(attachment, message))} +
` + : nothing + } ${ sendFailure @@ -2797,10 +2805,13 @@ export function createChatSurface( mimeType?: string; size?: number; content?: string; + preview?: string; artifactId?: string; + previewArtifactId?: string; } const localAttachmentUrls = new Map(); + const brokenUserImageSources = new Set(); function localContentUrl(a: UserAttachmentView): string | undefined { if (!a.content) return undefined; @@ -2818,13 +2829,28 @@ export function createChatSurface( } } - function userAttachmentBadge(a: UserAttachmentView): TemplateResult { + function userAttachmentBadge(a: UserAttachmentView, message: AgentMessage): TemplateResult | typeof nothing { const artifactHref = a.artifactId ? fileContentUrl(a.artifactId, a.fileName) : undefined; if (a.mimeType?.startsWith("image/")) { - const dataUrl = - a.content && (a.content.startsWith("data:") ? a.content : `data:${a.mimeType};base64,${a.content}`); - const download = !artifactHref || !browserRenderableImage(a.mimeType); - return chipBadge(FileImage, a.fileName, a.size, artifactHref ?? dataUrl ?? undefined, download); + const preview = a.preview?.startsWith("data:image/") ? a.preview : undefined; + const persistedPreview = a.previewArtifactId + ? `${fileContentUrl(a.previewArtifactId, a.fileName)}?preview=1` + : undefined; + const src = preview ?? persistedPreview; + if (browserRenderableImage(a.mimeType) && src && !brokenUserImageSources.has(src)) { + return html`Attached image { + brokenUserImageSources.add(src); + settledRowCache.delete(message as object); + redrawTranscript(); + }} + />`; + } + return imageChip(a.fileName, a.size, artifactHref); } if (inlineHtmlName(a.fileName, a.mimeType)) { let src = artifactHref; diff --git a/plugins/web-ui/src/composer.ts b/plugins/web-ui/src/composer.ts index 3bd9911276..787bed95b3 100644 --- a/plugins/web-ui/src/composer.ts +++ b/plugins/web-ui/src/composer.ts @@ -44,7 +44,7 @@ import { type QueuedRun, } from "./core-bridge"; import { errMessage } from "../../chassis/src/errors"; -import { fieldSelect, icon, modelMark } from "./ui"; +import { browserRenderableImage, fieldSelect, icon, modelMark } from "./ui"; import { EFFORT_LEVELS, defaultEffortForModel, @@ -253,6 +253,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { attachments: [] as Attachment[], error: "", processingFiles: false, + preparingImages: 0, dragging: false, openMenu: null as ComposerMenu | null, menuQuery: "", @@ -328,6 +329,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { pastedTextIds.clear(); composerState.error = ""; composerState.processingFiles = false; + composerState.preparingImages = 0; composerState.openMenu = null; slashActiveIndex = 0; composerState.slashDismissed = false; @@ -466,6 +468,43 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { } } + function stagedAttachment(attachment: Attachment, agent: Agent): TemplateResult { + const remove = (showTooltip: boolean) => html` + + `; + if (browserRenderableImage(attachment.mimeType) && attachment.preview?.startsWith("data:image/")) { + return html`${attachment.fileName}${remove(false)}`; + } + if (pastedTextIds.has(attachment.id)) { + return html` + + ${remove(true)} + `; + } + return html` + ${icon(Paperclip, 14)}${attachment.fileName}${remove(true)} + `; + } + function composerForm(agent: Agent, header: TemplateResult | typeof nothing = nothing): TemplateResult { const activeRuntimeConfig = getRuntimeConfig(scopeKey()); const selectedModel = currentModelOption(); @@ -512,7 +551,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { if (inputBlocked) placeholder = runtimePending ? "Loading runtime…" : "Approve or deny to continue"; else if (agent.state.isStreaming) placeholder = "Queue a message for after this turn…"; let composerNotice: TemplateResult | typeof nothing = nothing; - if (composerState.processingFiles) { + if (composerState.processingFiles && !composerState.preparingImages) { composerNotice = html`
Preparing files...
`; } else if (!approvalPauses.length && runtimePending) { composerNotice = composerState.error @@ -560,39 +599,19 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { : nothing } ${ - composerState.attachments.length + composerState.attachments.length || composerState.preparingImages ? html`
- ${composerState.attachments.map( - (a) => html` - - ${ - pastedTextIds.has(a.id) - ? html` - - ` - : html`${icon(Paperclip, 14)}${a.fileName}` - } - - - `, + ${composerState.attachments.map((attachment) => stagedAttachment(attachment, agent))} + ${Array.from( + { length: composerState.preparingImages }, + () => + html``, )}
` @@ -2189,10 +2208,126 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { return bytesToBase64(new Uint8Array(await file.arrayBuffer())); } + const IMAGE_PREVIEW_EDGE = 512; + const IMAGE_PREVIEW_SOURCE_BYTES = 10_000_000; + const IMAGE_PREVIEW_BYTES = 1_000_000; + const IMAGE_PREVIEW_SOURCE_PIXELS = 16_777_216; + let imagePreviewQueue = Promise.resolve(); + + async function imageDimensions(file: File): Promise<{ width: number; height: number } | undefined> { + const bytes = new Uint8Array(await file.slice(0, 262_144).arrayBuffer()); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const type = file.type.toLowerCase(); + const signature = (start: number, end: number) => String.fromCharCode(...bytes.subarray(start, end)); + if ( + (type === "image/png" || type === "image/apng") && + bytes.length >= 24 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return { width: view.getUint32(16), height: view.getUint32(20) }; + } + if (type === "image/gif" && bytes.length >= 10 && (signature(0, 6) === "GIF87a" || signature(0, 6) === "GIF89a")) { + return { width: view.getUint16(6, true), height: view.getUint16(8, true) }; + } + if ( + ["image/bmp", "image/x-ms-bmp"].includes(type) && + bytes.length >= 26 && + bytes[0] === 0x42 && + bytes[1] === 0x4d + ) { + return { width: Math.abs(view.getInt32(18, true)), height: Math.abs(view.getInt32(22, true)) }; + } + if (type === "image/webp" && bytes.length >= 30 && signature(0, 4) === "RIFF" && signature(8, 12) === "WEBP") { + let offset = 12; + while (offset + 8 <= bytes.length) { + const chunk = String.fromCharCode(...bytes.subarray(offset, offset + 4)); + const size = view.getUint32(offset + 4, true); + const data = offset + 8; + if (chunk === "VP8X" && data + 10 <= bytes.length) { + const width = 1 + bytes[data + 4]! + (bytes[data + 5]! << 8) + (bytes[data + 6]! << 16); + const height = 1 + bytes[data + 7]! + (bytes[data + 8]! << 8) + (bytes[data + 9]! << 16); + return { width, height }; + } + if (chunk === "VP8L" && data + 5 <= bytes.length && bytes[data] === 0x2f) { + const packed = view.getUint32(data + 1, true); + return { width: (packed & 0x3fff) + 1, height: ((packed >>> 14) & 0x3fff) + 1 }; + } + if (chunk === "VP8 " && data + 10 <= bytes.length) { + return { width: view.getUint16(data + 6, true) & 0x3fff, height: view.getUint16(data + 8, true) & 0x3fff }; + } + offset = data + size + (size % 2); + } + } + if (["image/jpeg", "image/jpg", "image/pjpeg"].includes(type) && bytes[0] === 0xff && bytes[1] === 0xd8) { + let offset = 2; + while (offset + 8 < bytes.length) { + if (bytes[offset++] !== 0xff) return undefined; + while (bytes[offset] === 0xff) offset++; + const marker = bytes[offset++]!; + const length = view.getUint16(offset); + if ([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf].includes(marker)) { + return { width: view.getUint16(offset + 5), height: view.getUint16(offset + 3) }; + } + if (length < 2 || offset + length > bytes.length) return undefined; + offset += length; + } + } + return undefined; + } + + async function boundedImagePreview(file: File): Promise { + if ( + !browserRenderableImage(file.type) || + file.size > IMAGE_PREVIEW_SOURCE_BYTES || + typeof createImageBitmap !== "function" + ) + return undefined; + let bitmap: ImageBitmap | undefined; + try { + const dimensions = await imageDimensions(file); + if ( + !dimensions || + dimensions.width <= 0 || + dimensions.height <= 0 || + dimensions.width > IMAGE_PREVIEW_SOURCE_PIXELS / dimensions.height + ) + return undefined; + const scale = Math.min(1, IMAGE_PREVIEW_EDGE / dimensions.width, IMAGE_PREVIEW_EDGE / dimensions.height); + const width = Math.max(1, Math.round(dimensions.width * scale)); + const height = Math.max(1, Math.round(dimensions.height * scale)); + bitmap = await createImageBitmap(file, { resizeWidth: width, resizeHeight: height, resizeQuality: "high" }); + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext("2d"); + if (!context) return undefined; + context.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + const preview = await new Promise((resolve) => canvas.toBlob(resolve, "image/webp", 0.82)); + if (!preview || preview.size > IMAGE_PREVIEW_BYTES) return undefined; + return `data:${preview.type};base64,${bytesToBase64(new Uint8Array(await preview.arrayBuffer()))}`; + } catch { + return undefined; + } finally { + bitmap?.close(); + } + } + async function loadAnyAttachment(file: File): Promise { + const preview = imagePreviewQueue.then(() => boundedImagePreview(file)); + imagePreviewQueue = preview.then( + () => undefined, + () => undefined, + ); try { const { loadAttachment } = await import("@earendil-works/pi-web-ui"); - return await loadAttachment(file); + return { ...(await loadAttachment(file)), preview: await preview }; } catch { return { id: `${file.name}_${Date.now()}_${Math.random()}`, @@ -2201,6 +2336,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { mimeType: file.type || "application/octet-stream", size: file.size, content: await fileToBase64(file), + preview: await preview, }; } } @@ -2268,10 +2404,11 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { ctx.chat.drawActiveChat(agent); return; } + const plan = planAdmission(files, folders.length); composerState.processingFiles = true; + composerState.preparingImages = plan.files.filter((file) => browserRenderableImage(file.type)).length; composerState.error = ""; ctx.chat.drawActiveChat(agent); - const plan = planAdmission(files, folders.length); try { const zipped: File[] = []; for (const folder of folders.slice(0, plan.folders)) zipped.push(await folderToZipFile(folder)); @@ -2287,6 +2424,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface { composerState.error = combineNote(plan.note ?? "", message); } finally { composerState.processingFiles = false; + composerState.preparingImages = 0; ctx.chat.drawActiveChat(agent); } } diff --git a/plugins/web-ui/src/conv-types.ts b/plugins/web-ui/src/conv-types.ts index 3460135d06..c222d7ec1a 100644 --- a/plugins/web-ui/src/conv-types.ts +++ b/plugins/web-ui/src/conv-types.ts @@ -114,6 +114,7 @@ interface ComposerState { attachments: Attachment[]; error: string; processingFiles: boolean; + preparingImages: number; dragging: boolean; openMenu: ComposerMenu | null; slashDismissed: boolean; diff --git a/plugins/web-ui/src/core-bridge.ts b/plugins/web-ui/src/core-bridge.ts index 3d6e9680d0..068648aa0a 100644 --- a/plugins/web-ui/src/core-bridge.ts +++ b/plugins/web-ui/src/core-bridge.ts @@ -48,12 +48,15 @@ interface PiAttachment { size: number; content: string; extractedText?: string; + preview?: string; } export interface CoreAttachment { name: string; mimetype: string; sizeBytes: number; blobId: string; + previewBlobId?: string; + previewMimetype?: string; } export const MAX_ATTACHMENT_BYTES = 1_000_000_000; @@ -636,15 +639,28 @@ function toHex(buf: ArrayBuffer): string { async function toCoreAttachment(a: PiAttachment): Promise { const bytes = attachmentBytes(a); - const sha256 = toHex(await crypto.subtle.digest("SHA-256", bytes as unknown as ArrayBuffer)); - const r = await webFetch(withBase(`/api/blobs?sha=${sha256}`), { - method: "POST", - headers: { "content-type": "application/octet-stream" }, - body: bytes as unknown as BodyInit, - }); - if (!r.ok) throw new ApiError(`attachment upload failed: HTTP ${r.status}`, r.status); - const { blobId, sizeBytes } = (await r.json()) as { blobId: string; sizeBytes: number }; - return { name: a.fileName, mimetype: a.mimeType, sizeBytes: sizeBytes ?? a.size, blobId }; + const upload = async (data: Uint8Array): Promise<{ blobId: string; sizeBytes: number }> => { + const sha256 = toHex(await crypto.subtle.digest("SHA-256", data as unknown as ArrayBuffer)); + const response = await webFetch(withBase(`/api/blobs?sha=${sha256}`), { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: data as unknown as BodyInit, + }); + if (!response.ok) throw new ApiError(`attachment upload failed: HTTP ${response.status}`, response.status); + return (await response.json()) as { blobId: string; sizeBytes: number }; + }; + const { blobId, sizeBytes } = await upload(bytes); + const previewData = /^data:(image\/(?:png|webp));base64,(.+)$/.exec(a.preview ?? ""); + const previewBytes = previewData ? base64ToBytes(previewData[2]!) : undefined; + const preview = + previewBytes && previewBytes.length <= 1_000_000 ? await upload(previewBytes).catch(() => undefined) : undefined; + return { + name: a.fileName, + mimetype: a.mimeType, + sizeBytes: sizeBytes ?? a.size, + blobId, + ...(preview ? { previewBlobId: preview.blobId, previewMimetype: previewData![1]! } : {}), + }; } export class ApiError extends Error { @@ -1633,6 +1649,7 @@ interface HistoryAttachment { mimeType: string; size?: number; artifactId?: string; + previewArtifactId?: string; } interface HistoryUserMessage { @@ -1876,7 +1893,13 @@ export function entriesToMessages(entries: SessionEntry[], model?: Model): text?: string; display?: string; callId?: string; - attachments?: Array<{ name?: string; mimetype?: string; sizeBytes?: number; artifactId?: string }>; + attachments?: Array<{ + name?: string; + mimetype?: string; + sizeBytes?: number; + artifactId?: string; + previewArtifactId?: string; + }>; files?: Array<{ name?: string; mimetype?: string; sizeBytes?: number; artifactId?: string }>; hidden?: boolean; steered?: boolean; @@ -1968,6 +1991,7 @@ export function entriesToMessages(entries: SessionEntry[], model?: Model): mimeType: a.mimetype ?? "application/octet-stream", ...(typeof a.sizeBytes === "number" ? { size: a.sizeBytes } : {}), ...(a.artifactId ? { artifactId: a.artifactId } : {}), + ...(a.previewArtifactId ? { previewArtifactId: a.previewArtifactId } : {}), })); } out.push(msg as AgentMessage); diff --git a/plugins/web-ui/src/main.ts b/plugins/web-ui/src/main.ts index b7f929c629..d0f226204c 100644 --- a/plugins/web-ui/src/main.ts +++ b/plugins/web-ui/src/main.ts @@ -14,6 +14,11 @@ import { } from "./sessions"; import { isPhone, onPhoneChange } from "./viewport"; +window.addEventListener("vite:preloadError", (event) => { + event.preventDefault(); + window.location.reload(); +}); + function closeComposerMenus(keepOpenWithin: Element | null): boolean { let changed = false; for (const conv of allConversations()) { diff --git a/plugins/web-ui/src/shared-session.ts b/plugins/web-ui/src/shared-session.ts index 9e3c078728..f2ecd86415 100644 --- a/plugins/web-ui/src/shared-session.ts +++ b/plugins/web-ui/src/shared-session.ts @@ -1,12 +1,12 @@ import "./shell.css"; import "@mariozechner/mini-lit/dist/ThemeToggle.js"; -import { html, render } from "lit"; +import { html, render, type TemplateResult } from "lit"; import { Lock, ArrowUpRight, Check, Copy, File, FileImage } from "lucide"; import { createTranscriptViewport } from "./transcript-viewport"; import { decorateTextCodeBlocks } from "./text-code"; import { markdown } from "./message-markdown"; import { installMarkdownSanitizer } from "./markdown-sanitize"; -import { brandName, brandMark, chipBadge, icon, copyText } from "./ui"; +import { brandName, brandMark, browserRenderableImage, chipBadge, icon, copyText } from "./ui"; interface SharedTranscript { createdAt: number; @@ -14,15 +14,24 @@ interface SharedTranscript { messages: Array<{ role: "user" | "assistant"; text: string; - attachments?: Array<{ id: string; name: string; mimetype: string; sizeBytes: number }>; + attachments?: Array<{ + id: string; + name: string; + mimetype: string; + sizeBytes: number; + inlinePreview?: boolean; + previewId?: string; + }>; }>; } installMarkdownSanitizer({ shared: true }); const transcript: SharedTranscript | null = JSON.parse(document.getElementById("shared-transcript")!.textContent!); const base = (import.meta as unknown as { env: { BASE_URL: string } }).env.BASE_URL; -render( - html` +const failedImageSources = new Set(); + +function sharedConversation(): TemplateResult { + return html`
${message.attachments.map((file) => { const href = `${location.pathname}/files/${encodeURIComponent(file.id)}`; - const inlineImage = /^image\/(png|jpeg|gif|webp|avif)$/.test(file.mimetype); - if (inlineImage && message.role !== "user") { + const imageSrc = file.previewId + ? `${location.pathname}/files/${encodeURIComponent(file.previewId)}?inline=1` + : `${href}?inline=1`; + const inlineImage = + browserRenderableImage(file.mimetype) && + (message.role !== "user" || + (file.inlinePreview === true && Boolean(file.previewId))); + const failedImage = failedImageSources.has(imageSrc); + if (message.role === "user" && inlineImage && !failedImage) { + return html`Attached image { + failedImageSources.add(imageSrc); + draw(); + }} + />`; + } + if (inlineImage && message.role !== "user" && !failedImage) { return html`${file.name}${file.name} { + failedImageSources.add(imageSrc); + draw(); + }} />`; } return chipBadge( inlineImage ? FileImage : File, file.name, file.sizeBytes, - inlineImage ? `${href}?inline=1` : href, - !inlineImage, + inlineImage && !failedImage ? imageSrc : href, + !inlineImage || failedImage, ); })}
` @@ -92,9 +131,14 @@ render( ${icon(Lock, 12)}${transcript ? `Shared snapshot · ${new Date(transcript.createdAt).toLocaleDateString()} · ${transcript.audience === "external" ? "Anyone with the link" : "Organization only"}` : "Shared conversation"} - `, - document.getElementById("app")!, -); + `; +} + +function draw(): void { + render(sharedConversation(), document.getElementById("app")!); +} + +draw(); const viewport = createTranscriptViewport(); requestAnimationFrame(() => { diff --git a/plugins/web-ui/src/shell.css b/plugins/web-ui/src/shell.css index a58516b6d8..22266e78ac 100644 --- a/plugins/web-ui/src/shell.css +++ b/plugins/web-ui/src/shell.css @@ -2040,6 +2040,13 @@ qm-markdown .tok-in { gap: 6px; margin-top: 9px; } +.attachment-strip { + margin-inline: var(--composer-input-pad-inline); + margin-bottom: 4px; +} +.composer-wrap:has(.composer-input:dir(rtl)) .attachment-strip { + direction: rtl; +} .queued-strip, .chat-bottom-dock > .bg-activity { position: relative; @@ -2143,6 +2150,78 @@ qm-markdown .tok-in { .file-chip small { color: var(--muted-foreground); } +.image-preview { + position: relative; + display: inline-flex; + width: 70px; + height: 70px; + border-radius: 8px; + background: var(--secondary); + overflow: hidden; +} +.image-preview img { + width: 100%; + height: 100%; + object-fit: cover; +} +.image-preview-loading { + align-items: center; + justify-content: center; + background: color-mix(in srgb, var(--muted-foreground) 10%, var(--background)); +} +.image-preview-spinner { + width: 18px; + height: 18px; + border: 2px solid color-mix(in srgb, var(--muted-foreground) 24%, transparent); + border-top-color: var(--muted-foreground); + border-radius: 50%; + animation: image-preview-spin 0.7s linear infinite; +} +@keyframes image-preview-spin { + to { + transform: rotate(360deg); + } +} +@media (prefers-reduced-motion: reduce) { + .image-preview-spinner { + animation: none; + } +} +.image-preview .chip-x { + position: absolute; + top: 4px; + right: 4px; + width: 20px; + height: 20px; + margin: 0; + border-radius: 5px; + background: color-mix(in srgb, var(--background) 44%, transparent); + color: var(--muted-foreground); + transition: + opacity 0.12s ease, + background 0.12s ease, + color 0.12s ease; +} +.image-preview .chip-x:hover, +.image-preview .chip-x:focus-visible { + background: var(--secondary); + color: var(--foreground); +} +@media (hover: hover) and (pointer: fine) { + .image-preview .chip-x { + opacity: 0; + } + .image-preview:hover .chip-x, + .image-preview:focus-within .chip-x { + opacity: 1; + } +} +@media (hover: none) { + .image-preview .chip-x { + width: 24px; + height: 24px; + } +} .file-image { display: inline-block; @@ -2161,6 +2240,50 @@ qm-markdown .tok-in { height: auto; object-fit: contain; } +.user-image-attachment { + display: block; + max-width: min(360px, 100%); + max-height: 360px; + width: auto; + height: auto; + border-radius: 10px; + object-fit: contain; +} +a.user-image-attachment { + display: inline-block; + overflow: hidden; + line-height: 0; +} +a.user-image-attachment > img { + display: block; + max-width: 100%; + max-height: 360px; + width: auto; + height: auto; + object-fit: contain; +} +.message-files:has(> .user-image-attachment:first-child:nth-last-child(2) + .user-image-attachment:last-child) { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + width: 324px; + max-width: 100%; + gap: 4px; +} +.message-files:has(> .user-image-attachment:first-child:nth-last-child(2) + .user-image-attachment:last-child) + > .user-image-attachment { + width: 100%; + max-width: none; + max-height: none; + aspect-ratio: 1; + object-fit: cover; +} +.message-files:has(> .user-image-attachment:first-child:nth-last-child(2) + .user-image-attachment:last-child) + > a.user-image-attachment + > img { + width: 100%; + height: 100%; + object-fit: cover; +} .file-html-unfurl { display: flex; flex-direction: column; @@ -3230,6 +3353,7 @@ qm-markdown .tok-in { --composer-pad-block: 12px; --composer-pad-inline: 12px; --composer-pad-end: var(--composer-pad-inline); + --composer-input-pad-inline: 8px; position: relative; z-index: 1; width: min(var(--content-w), calc(100% - 32px)); @@ -3259,7 +3383,7 @@ qm-markdown .tok-in { font: inherit; font-size: var(--composer-font-size); line-height: 1.5; - padding: 7px 8px 8px; + padding: 7px var(--composer-input-pad-inline) 8px; box-sizing: border-box; } .composer-input::placeholder { @@ -10529,6 +10653,7 @@ h3.ambient-field-label { --composer-pad-block: 8px; --composer-pad-inline: 10px; --composer-pad-end: 8px; + --composer-input-pad-inline: 10px; padding: var(--composer-pad-block) var(--composer-pad-end) var(--composer-pad-block) var(--composer-pad-inline); border-radius: 20px; } @@ -10536,7 +10661,7 @@ h3.ambient-field-label { line-height: 1.4; min-height: 40px; max-height: 32vh; - padding: 12px 10px 6px; + padding: 12px var(--composer-input-pad-inline) 6px; } .composer-wrap.compact .composer-toolbar { flex-wrap: nowrap; diff --git a/plugins/web-ui/src/ui.ts b/plugins/web-ui/src/ui.ts index 1135c6bd5c..5719367076 100644 --- a/plugins/web-ui/src/ui.ts +++ b/plugins/web-ui/src/ui.ts @@ -316,24 +316,7 @@ export function formatBytes(bytes: number): string { return `${(bytes / 1024 / 1024).toFixed(1)} MB`; } -const RENDERABLE_IMAGE_TYPES = new Set([ - "image/png", - "image/jpeg", - "image/jpg", - "image/pjpeg", - "image/gif", - "image/webp", - "image/avif", - "image/bmp", - "image/apng", - "image/x-icon", - "image/vnd.microsoft.icon", - "image/x-ms-bmp", -]); - -export function browserRenderableImage(mimeType?: string): boolean { - return RENDERABLE_IMAGE_TYPES.has((mimeType ?? "").split(";")[0]!.trim().toLowerCase()); -} +export { browserRenderableImage } from "../../chassis/src/image-mime.ts"; const copyFeedback = new WeakMap>(); diff --git a/plugins/web-ui/test/attachment-limits.test.ts b/plugins/web-ui/test/attachment-limits.test.ts index b89c54f20d..6808f2be19 100644 --- a/plugins/web-ui/test/attachment-limits.test.ts +++ b/plugins/web-ui/test/attachment-limits.test.ts @@ -76,6 +76,29 @@ test("one failing blob degrades that file only; the rest upload and the send sur assert.equal(skipped[0]!.note, oversizeAttachmentNote("rejected.txt"), "a 413 reads as the size limit, not raw HTTP"); }); +test("Safari PNG preview fallbacks are uploaded with their actual MIME type", async () => { + let call = 0; + globalThis.fetch = (async () => { + call++; + return new Response(JSON.stringify({ blobId: `b${call}`, sizeBytes: 4 }), { status: 200 }); + }) as typeof fetch; + const { uploaded, skipped } = await uploadAttachments([ + { + id: "image-1", + type: "image", + fileName: "image.png", + mimeType: "image/png", + size: 3, + content: "YWJj", + preview: "data:image/png;base64,cHJldmlldw==", + }, + ]); + assert.deepEqual(skipped, []); + assert.equal(call, 2); + assert.equal(uploaded[0]!.previewBlobId, "b2"); + assert.equal(uploaded[0]!.previewMimetype, "image/png"); +}); + test("a non-413 upload failure is noted per file with its reason", async () => { globalThis.fetch = (async () => { throw new Error("network unreachable"); diff --git a/plugins/web-ui/test/composer-stopping.test.ts b/plugins/web-ui/test/composer-stopping.test.ts index cde327a48c..4a80dffeeb 100644 --- a/plugins/web-ui/test/composer-stopping.test.ts +++ b/plugins/web-ui/test/composer-stopping.test.ts @@ -165,10 +165,58 @@ test("stopping blocks send and queue through render, input, keyboard, and form w composer!.state.attachments = [attachment]; draw(); assert.equal(button(".send-btn").disabled, true, "attachment-only send remains blocked"); + assert.equal(host.querySelector(".attachment-strip .file-chip span")?.textContent, "draft.txt"); + assert.equal(host.querySelector(".attachment-strip .image-preview"), null); host.querySelector("form")!.dispatchEvent(new dom.window.Event("submit", { cancelable: true })); assert.deepEqual(composer!.state.attachments, [attachment], "stopping preserves attachments"); assert.equal(requests.length, 0); + composer!.state.attachments = [ + { + id: "test-image", + type: "image", + fileName: "tiny.png", + mimeType: "image/png", + size: 1, + content: "iVBORw0KGgo=", + preview: "data:image/webp;base64,cHJldmlldw==", + }, + ]; + draw(); + const preview = host.querySelector(".attachment-strip .image-preview img"); + assert.equal(preview?.src, "data:image/webp;base64,cHJldmlldw=="); + assert.equal(preview?.alt, "tiny.png"); + preview?.dispatchEvent(new dom.window.MouseEvent("mouseenter", { bubbles: true })); + assert.notEqual(dom.window.document.querySelector(".qm-tooltip.visible")?.textContent, "tiny.png"); + assert.equal(host.querySelector(".attachment-strip .file-chip"), null); + const imageRemove = host.querySelector('.image-preview [aria-label="Remove attachment"]'); + assert.ok(imageRemove); + imageRemove.dispatchEvent(new dom.window.MouseEvent("mouseenter", { bubbles: true })); + assert.notEqual(dom.window.document.querySelector(".qm-tooltip.visible")?.textContent, "Remove"); + composer!.state.attachments = [ + { + id: "unbounded-image", + type: "image", + fileName: "original.png", + mimeType: "image/png", + size: 1, + content: "iVBORw0KGgo=", + }, + ]; + draw(); + assert.equal(host.querySelector(".attachment-strip .image-preview"), null); + assert.equal(host.querySelector(".attachment-strip .file-chip span")?.textContent, "original.png"); composer!.state.attachments = []; + composer!.state.preparingImages = 2; + composer!.state.processingFiles = true; + draw(); + const placeholders = host.querySelectorAll( + '.image-preview.image-preview-loading[aria-label="Preparing image preview"]', + ); + assert.equal(placeholders.length, 2); + assert.equal(host.querySelectorAll(".image-preview-spinner").length, 2); + assert.equal(host.querySelector(".composer-note"), null); + composer!.state.preparingImages = 0; + composer!.state.processingFiles = false; stopping = false; composer!.state.draft = "Resume sending"; draw(); diff --git a/plugins/web-ui/test/css-vars.test.ts b/plugins/web-ui/test/css-vars.test.ts index be87db6564..63e52cce83 100644 --- a/plugins/web-ui/test/css-vars.test.ts +++ b/plugins/web-ui/test/css-vars.test.ts @@ -24,6 +24,77 @@ test("every no-fallback var() in shell.css names a property something defines", assert.deepEqual([...dead], [], "var() references that nothing defines (add the property or a fallback)"); }); +test("composer attachments share the textarea text inset", () => { + assert.match(shellCss, /--composer-input-pad-inline:\s*8px;/); + assert.match(shellCss, /\.composer-input \{[\s\S]*?padding:\s*7px var\(--composer-input-pad-inline\) 8px;/); + assert.match( + shellCss, + /\.attachment-strip \{\s*margin-inline:\s*var\(--composer-input-pad-inline\);\s*margin-bottom:\s*4px;\s*\}/, + ); + assert.match( + shellCss, + /\.composer-wrap:has\(\.composer-input:dir\(rtl\)\) \.attachment-strip \{\s*direction:\s*rtl;/, + ); + assert.match(shellCss, /--composer-input-pad-inline:\s*10px;/); +}); + +test("composer image previews are 25% larger", () => { + const preview = shellCss.match(/\.image-preview \{[^}]+\}/)?.[0] ?? ""; + assert.match(preview, /width:\s*70px;\s*height:\s*70px;/); + assert.doesNotMatch(preview, /border:/); + assert.match( + shellCss, + /\.image-preview-loading \{\s*align-items:\s*center;\s*justify-content:\s*center;\s*background:\s*color-mix\(in srgb, var\(--muted-foreground\) 10%, var\(--background\)\);/, + ); + assert.match( + shellCss, + /\.image-preview-spinner \{[^}]*width:\s*18px;[^}]*height:\s*18px;[^}]*animation:\s*image-preview-spin 0\.7s linear infinite;/, + ); + assert.match( + shellCss, + /@media \(prefers-reduced-motion: reduce\) \{\s*\.image-preview-spinner \{\s*animation:\s*none;/, + ); +}); + +test("sent user images have a passive medium presentation", () => { + const image = shellCss.match(/\.user-image-attachment \{[^}]+\}/)?.[0] ?? ""; + assert.match(image, /max-width:\s*min\(360px, 100%\);/); + assert.match(image, /max-height:\s*360px;/); + assert.match(image, /border-radius:\s*10px;/); + assert.doesNotMatch(image, /border:|cursor:/); +}); + +test("two sent user images form a compact side-by-side pair", () => { + const pair = + shellCss.match(/\.message-files:has\([\s\S]*?\.user-image-attachment:last-child\s*\) \{[^}]+\}/)?.[0] ?? ""; + assert.match(pair, /display:\s*grid;/); + assert.match(pair, /grid-template-columns:\s*repeat\(2, minmax\(0, 1fr\)\);/); + assert.match(pair, /width:\s*324px;/); + assert.match(pair, /max-width:\s*100%;/); + assert.match(pair, /gap:\s*4px;/); + const images = shellCss.match(/\.message-files:has\([\s\S]*?> \.user-image-attachment \{[^}]+\}/)?.[0] ?? ""; + assert.match(images, /aspect-ratio:\s*1;/); + assert.match(images, /object-fit:\s*cover;/); +}); + +test("image remove actions stay compact", () => { + const imageAction = shellCss.match(/\.image-preview \.chip-x \{[^}]+\}/)?.[0] ?? ""; + assert.match(imageAction, /width:\s*20px;/); + assert.match(imageAction, /height:\s*20px;/); + assert.match(imageAction, /border-radius:\s*5px;/); + assert.match(imageAction, /background:\s*color-mix\(in srgb, var\(--background\) 44%, transparent\);/); + assert.match(imageAction, /color:\s*var\(--muted-foreground\);/); + assert.match(imageAction, /opacity 0\.12s ease,/); + assert.match(imageAction, /background 0\.12s ease,/); + assert.match(imageAction, /color 0\.12s ease;/); + assert.match( + shellCss, + /\.image-preview \.chip-x:hover,\s*\.image-preview \.chip-x:focus-visible \{\s*background:\s*var\(--secondary\);\s*color:\s*var\(--foreground\);/, + ); + assert.match(shellCss, /\.image-preview:hover \.chip-x,\s*\.image-preview:focus-within \.chip-x \{\s*opacity:\s*1;/); + assert.match(shellCss, /@media \(hover: none\) \{\s*\.image-preview \.chip-x \{\s*width:\s*24px;\s*height:\s*24px;/); +}); + test("colored session actions keep their row hue at rest and on hover", () => { const variables = shellCss.match(/\.session-row\.colored \{[^}]+\}/)?.[0] ?? ""; assert.match(variables, /--session-action-hover:\s*color-mix\([^;]+var\(--session-color\)/); diff --git a/plugins/web-ui/test/renderable-image-source.test.ts b/plugins/web-ui/test/renderable-image-source.test.ts index 2ba8f494d7..8f45d01652 100644 --- a/plugins/web-ui/test/renderable-image-source.test.ts +++ b/plugins/web-ui/test/renderable-image-source.test.ts @@ -2,11 +2,11 @@ import { readFileSync } from "node:fs"; import { test } from "node:test"; import assert from "node:assert/strict"; -const ui = readFileSync(new URL("../src/ui.ts", import.meta.url), "utf8"); +const imageMime = readFileSync(new URL("../../chassis/src/image-mime.ts", import.meta.url), "utf8"); const chat = readFileSync(new URL("../src/chat.ts", import.meta.url), "utf8"); test("SVG attachments fall back to a visible download chip", () => { - const renderableTypes = ui.match(/const RENDERABLE_IMAGE_TYPES = new Set\(\[([\s\S]*?)\]\);/)?.[1] ?? ""; + const renderableTypes = imageMime.match(/const RENDERABLE_IMAGE_TYPES = new Set\(\[([\s\S]*?)\]\);/)?.[1] ?? ""; assert.doesNotMatch(renderableTypes, /image\/svg\+xml/); assert.match(chat, /if \(!browserRenderableImage\(file\.mimetype\)\) return imageChip/); }); diff --git a/plugins/web-ui/test/session-jump.test.ts b/plugins/web-ui/test/session-jump.test.ts index 95ceaf279b..2e0f02aee6 100644 --- a/plugins/web-ui/test/session-jump.test.ts +++ b/plugins/web-ui/test/session-jump.test.ts @@ -30,6 +30,13 @@ test("the session jump hotkey is registered at boot", () => { assert.match(main, /registerSessionJumpHotkeys\(\)/); }); +test("stale dynamic modules reload the current app version", () => { + assert.match( + main, + /window\.addEventListener\("vite:preloadError", \(event\) => \{\s*event\.preventDefault\(\);\s*window\.location\.reload\(\);/, + ); +}); + test("sidebar session rows expose the anchor the hotkey targets", () => { const row = sessions.match(/function sessionRow\([^]*?\n\}/)?.[0] ?? ""; assert.match(row, /class="session"/, "sessionRow renders an a.session anchor"); diff --git a/plugins/web-ui/test/user-message-cap.test.ts b/plugins/web-ui/test/user-message-cap.test.ts index 1ef0bb1e04..c4b120f6c9 100644 --- a/plugins/web-ui/test/user-message-cap.test.ts +++ b/plugins/web-ui/test/user-message-cap.test.ts @@ -4,6 +4,7 @@ import test from "node:test"; const css = readFileSync(new URL("../src/shell.css", import.meta.url), "utf8"); const chat = readFileSync(new URL("../src/chat.ts", import.meta.url), "utf8"); +const composer = readFileSync(new URL("../src/composer.ts", import.meta.url), "utf8"); const shared = readFileSync(new URL("../src/shared-session.ts", import.meta.url), "utf8"); test("only the collapsed pinned prompt is capped and overflow clips instead of nesting scrollbars", () => { @@ -25,25 +26,68 @@ test("the scroller is the size container the cap measures", () => { assert.doesNotMatch(chat, /--chat-viewport/); }); -test("images a user attached render as chips, not inline, in the live chat", () => { +test("images a user attached render as passive images in the live chat", () => { const fn = chat.match(/function userAttachmentBadge\([\s\S]*?\n {2}\}/)?.[0] ?? ""; assert.match(fn, /startsWith\("image\/"\)/); + assert.match(fn, /a\.preview\?\.startsWith\("data:image\/"\)/); + assert.match(fn, /const src = preview \?\? persistedPreview;/); + assert.doesNotMatch(fn, /const src = preview \?\? artifactHref/); + assert.match(fn, /alt="Attached image"/); + assert.match(fn, /!brokenUserImageSources\.has\(src\)/); + assert.match(fn, /brokenUserImageSources\.add\(src\);/); + assert.match(fn, /settledRowCache\.delete\(message as object\);/); + assert.match(fn, /\?preview=1/); + assert.match(fn, /return imageChip\(a\.fileName, a\.size, artifactHref\);/); + assert.doesNotMatch(fn, /const dataUrl|artifactHref \?\? dataUrl/); + assert.doesNotMatch(fn, /chipBadge\(FileImage|tip\(|download|title=/); +}); + +test("local image previews are bounded before they reach an image element", () => { + const preview = composer.match(/async function boundedImagePreview\([\s\S]*?\n {2}\}/)?.[0] ?? ""; + const staged = composer.match(/function stagedAttachment\([\s\S]*?\n {2}\}/)?.[0] ?? ""; + assert.match(composer, /const IMAGE_PREVIEW_EDGE = 512;/); + assert.match(composer, /const IMAGE_PREVIEW_SOURCE_BYTES = 10_000_000;/); + assert.match(composer, /const IMAGE_PREVIEW_BYTES = 1_000_000;/); + assert.match(composer, /const IMAGE_PREVIEW_SOURCE_PIXELS = 16_777_216;/); + assert.match(composer, /const preview = imagePreviewQueue\.then\(\(\) => boundedImagePreview\(file\)\);/); + assert.match(preview, /dimensions\.width > IMAGE_PREVIEW_SOURCE_PIXELS \/ dimensions\.height/); + assert.match(preview, /createImageBitmap\(file, \{ resizeWidth: width, resizeHeight: height/); + assert.match(preview, /canvas\.width = width/); + assert.match(preview, /canvas\.height = height/); + assert.match(preview, /preview\.size > IMAGE_PREVIEW_BYTES/); + assert.match(staged, /attachment\.preview\?\.startsWith\("data:image\/"\)/); + assert.match(staged, /icon\(X, 11\)/); + assert.doesNotMatch(staged, /attachment\.content/); + assert.match( + composer, + /composerState\.preparingImages = plan\.files\.filter\(\(file\) => browserRenderableImage\(file\.type\)\)\.length;/, + ); assert.match( - fn, - /return chipBadge\(FileImage, a\.fileName, a\.size, artifactHref \?\? dataUrl \?\? undefined, download\);/, + composer, + /class="image-preview image-preview-loading"\s*role="status"\s*aria-label="Preparing image preview"/, ); - assert.doesNotMatch(fn, / { +test("images a user attached render as passive images on the share page", () => { const files = shared.match(/message\.attachments\.map\(\(file\) => \{[\s\S]*?\n\s*\}\)\}/)?.[0] ?? ""; - assert.match(files, /if \(inlineImage && message\.role !== "user"\) \{\s*return html` { diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.ts index 4e2d0f9d3b..6bd80b8eb3 100644 --- a/src/api/app-sessions.ts +++ b/src/api/app-sessions.ts @@ -2,7 +2,7 @@ import { sessionTreeRoot, sessionTreeRunCount, SUBAGENT_TREE_RUN_CAP } from "../ import type { PendingApprovalRecord } from "../types.ts"; import { orgId as orgIdOf } from "../config.ts"; import { parseScopeId, scopeId } from "../types.ts"; -import { fileArtifactId, artifactPath } from "../files/file-artifact-store.ts"; +import { fileArtifactId, artifactPath, isPreviewArtifactPath } from "../files/file-artifact-store.ts"; import { entryWithinTenure, transcriptEntries, windowedTranscript } from "../sessions/session-store.ts"; import { createTranscriptSource } from "../harness/tape-projection.ts"; import { appendCoverageImport } from "../harness/replay.ts"; @@ -370,9 +370,10 @@ export function createSessionMethods( return toFileItem(artifact); }, - async openFileForViewer(id, principalId) { - const art = await deps.files.get(id); + async openFileForViewer(id, principalId, opts) { + const art = await deps.files.get(id, opts?.preview ? { includeDisabled: true } : undefined); if (!art) return null; + if (opts?.preview && !isPreviewArtifactPath(art.path)) return null; const myScopes = await currentResourceScopesForViewer(principalId); let allowed = myScopes.includes(art.ownerScopeId); if (!allowed) { @@ -380,7 +381,7 @@ export function createSessionMethods( allowed = grants.some((g) => myScopes.includes(g.granteeScopeId)); } if (!allowed) return null; - const opened = await deps.files.open(id); + const opened = await deps.files.open(id, opts?.preview ? { includeDisabled: true } : undefined); if (!opened) return null; return { name: art.name, mimetype: art.mimetype, sizeBytes: opened.sizeBytes, stream: opened.stream }; }, diff --git a/src/api/app-types.ts b/src/api/app-types.ts index c465d80f3d..e079c9808b 100644 --- a/src/api/app-types.ts +++ b/src/api/app-types.ts @@ -392,7 +392,7 @@ export interface App { authorizesCapabilityScope( claims: Pick, ): Promise; - openFileForViewer(id: string, principalId: string): Promise; + openFileForViewer(id: string, principalId: string, opts?: { preview?: boolean }): Promise; grant(g: Grant): Promise; revokeGrant(ownerScopeId: ScopeId, ref: string, granteeScopeId: ScopeId, revokedBy: string): Promise; promoteSkill(id: string, targetScopeId: ScopeId, actorId: string, liveActor: boolean): Promise; diff --git a/src/api/routes/session-sharing.ts b/src/api/routes/session-sharing.ts index c3670ffc71..2f2ec45bbc 100644 --- a/src/api/routes/session-sharing.ts +++ b/src/api/routes/session-sharing.ts @@ -7,6 +7,7 @@ import { type SharedMessage, } from "../../sessions/session-share.ts"; import { MAX_ATTACHMENT_BYTES } from "../../core/attachments.ts"; +import { browserRenderableImage } from "../../../plugins/chassis/src/image-mime.ts"; import { pipeToResponse, sendJson } from "../http.ts"; import { audit, isObj } from "./shared.ts"; import type { ApiCtx, Route } from "./route.ts"; @@ -43,7 +44,8 @@ async function createShare(ctx: ApiCtx): Promise { } const projected = sharedMessages(source.entries, deliveredAttachments); if (!projected.length) return sendJson(res, 400, { error: "empty_conversation" }); - const ids = [...new Set(projected.flatMap((m) => m.attachmentIds ?? []))]; + const previewIds = new Set(projected.flatMap((m) => m.inlinePreviewIds ?? [])); + const ids = [...new Set([...projected.flatMap((m) => m.attachmentIds ?? []), ...previewIds])]; if (ids.length > 100 || Buffer.byteLength(JSON.stringify(projected)) > 2_000_000) return sendJson(res, 413, { error: "share_too_large" }); const files: SessionShare["files"] = []; @@ -51,15 +53,19 @@ async function createShare(ctx: ApiCtx): Promise { const pending: Array<{ sourceId: string; name: string; mimetype: string; data: Buffer }> = []; let totalBytes = 0; for (const id of ids) { - const file = await app.openFileForViewer(id, viewer); - if (!file) + const preview = previewIds.has(id); + const file = await app.openFileForViewer(id, viewer, preview ? { preview: true } : undefined); + if (!file) { + if (preview) continue; return sendJson(res, 409, { error: "attachment_unavailable", message: "An attachment is no longer available to share.", }); + } const maxBytes = Math.min(MAX_ATTACHMENT_BYTES, 100 * 1024 * 1024 - totalBytes); if (file.sizeBytes > maxBytes) { file.stream.destroy(); + if (preview) continue; return sendJson(res, 413, { error: "attachments_too_large" }); } try { @@ -68,20 +74,38 @@ async function createShare(ctx: ApiCtx): Promise { pending.push({ sourceId: id, name: file.name, mimetype: file.mimetype, data: collected.data }); } catch { file.stream.destroy(); + if (preview) continue; return sendJson(res, 409, { error: "attachment_unavailable", message: "An attachment could not be copied." }); } } for (const file of pending) { const stored = await deps.sessionShareBytes.put(file.data, { maxBytes: MAX_ATTACHMENT_BYTES }); - const attachment = { id: randomUUID(), name: file.name, mimetype: file.mimetype, sizeBytes: stored.sizeBytes }; + const attachment = { + id: randomUUID(), + name: file.name, + mimetype: file.mimetype, + sizeBytes: stored.sizeBytes, + ...(previewIds.has(file.sourceId) ? { inlinePreview: true } : {}), + }; attachments.set(file.sourceId, attachment); files.push({ ...attachment, blobKey: stored.blobKey }); } - const messages: SharedMessage[] = projected.map(({ role, text, attachmentIds }) => ({ - role, - text, - ...(attachmentIds?.length ? { attachments: attachmentIds.map((id) => attachments.get(id)!) } : {}), - })); + const messages: SharedMessage[] = projected.map(({ role, text, attachmentIds, previewPairs }) => { + const previews = new Map(previewPairs?.map((pair) => [pair.attachmentId, attachments.get(pair.previewId)?.id])); + return { + role, + text, + ...(attachmentIds?.length + ? { + attachments: attachmentIds.map((id) => { + const attachment = attachments.get(id)!; + const previewId = previews.get(id); + return previewId ? { ...attachment, inlinePreview: true, previewId } : attachment; + }), + } + : {}), + }; + }); const share: SessionShare = { token: randomUUID(), sessionId: params.id!, @@ -132,7 +156,7 @@ async function readShare(ctx: ApiCtx): Promise { const file = share.files.find((file) => file.id === params.fileId); const opened = file && (await deps.sessionShareBytes?.open(file.blobKey)); if (!file || !opened) return sendJson(res, 404, { error: "not_found" }); - const inline = url.searchParams.get("inline") === "1" && /^image\/(png|jpeg|gif|webp|avif)$/.test(file.mimetype); + const inline = url.searchParams.get("inline") === "1" && browserRenderableImage(file.mimetype); res.writeHead(200, { "content-type": inline ? file.mimetype : "application/octet-stream", "content-length": String(opened.sizeBytes), diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.ts index 28783544f3..c71baa6ac3 100644 --- a/src/api/routes/surface.ts +++ b/src/api/routes/surface.ts @@ -300,11 +300,15 @@ async function getSessionBackgroundOutput(ctx: ApiCtx): Promise { } async function getFileContent(ctx: ApiCtx): Promise { - const { res, app, capability, actor } = ctx; + const { res, app, capability, actor, url } = ctx; const id = ctx.params.id!; const viewer = capability?.actorId ?? actor?.p; if (!viewer) return sendJson(res, 401, { error: "capability_required" }); - const opened = await app.openFileForViewer(id, viewer); + const opened = await app.openFileForViewer( + id, + viewer, + url.searchParams.get("preview") === "1" ? { preview: true } : undefined, + ); if (!opened) return sendJson(res, 404, { error: "not_found" }); res.writeHead(200, { "content-type": contentTypeWithUtf8Charset(opened.mimetype || "application/octet-stream"), diff --git a/src/core/attachments.ts b/src/core/attachments.ts index 6a33e68548..4cc531b0c0 100644 --- a/src/core/attachments.ts +++ b/src/core/attachments.ts @@ -10,12 +10,17 @@ import type { } from "../types.ts"; import { hasParentPathSegment, type Sandbox, type SandboxHandle } from "../sandbox/sandbox.ts"; import { MAX_BLOB_BYTES, collectBlob, type BlobTransferStore } from "../persistence/blob-transfer.ts"; -import { fileArtifactId, type FileArtifactStore, type FileDirection } from "../files/file-artifact-store.ts"; +import { + fileArtifactId, + previewArtifactPath, + type FileArtifactStore, + type FileDirection, +} from "../files/file-artifact-store.ts"; import { parseRef } from "../acl/resource-ref.ts"; import { swallowAs } from "../util/errors.ts"; import { hashId } from "../util/crypto.ts"; import type { SecurityScreenVerdict } from "../security/security-posture.ts"; -import { downscaleVisionImage } from "./image-downscale.ts"; +import { downscaleVisionImage, sniffImageDimensions } from "./image-downscale.ts"; export const INBOX_DIR = "inbox"; export const SHARED_DIR = "shared"; @@ -28,6 +33,10 @@ export function turnFileId(runId?: string, attempt = 1, now = Date.now()): strin export const MAX_ATTACHMENT_BYTES = MAX_BLOB_BYTES; const VISION_MIME_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); +const PREVIEW_FORMAT_BY_MIME: ReadonlyMap = new Map([ + ["image/png", "png"], + ["image/webp", "webp"], +] as const); export const MAX_VISION_IMAGE_BYTES = 5_000_000; @@ -144,12 +153,13 @@ async function registerArtifact( name: string, mimetype: string, bytes: Uint8Array, + preview = false, ): Promise< { id: string; path: string; ownerScopeId: ScopeId; direction: FileDirection; created: boolean } | undefined > { try { const id = fileArtifactId(reg.seed, direction, batchIndex); - const path = `artifacts/${id}/${name}`; + const path = preview ? previewArtifactPath(id, name) : `artifacts/${id}/${name}`; const { created } = await reg.store.put({ id, ownerScopeId: reg.ownerScopeId, @@ -161,6 +171,7 @@ async function registerArtifact( direction, ...(reg.createdInScope ? { createdInScope: reg.createdInScope } : {}), maxBytes: MAX_ATTACHMENT_BYTES, + enabled: !preview, }); const registered = { id, path, ownerScopeId: reg.ownerScopeId, direction, created }; await reg.onRegistered?.(registered); @@ -353,6 +364,40 @@ export async function materializeInbound( const registered = register ? await registerArtifact(register, "in", metas.length, name, mimetype, bytes) : undefined; + let previewArtifactId: string | undefined; + const previewMimetype = a.previewMimetype; + const previewFormat = PREVIEW_FORMAT_BY_MIME.get(previewMimetype ?? ""); + if (register && a.previewBlobId && previewMimetype && previewFormat) { + let preview: Awaited> = null; + try { + preview = await transfer.open(a.previewBlobId); + if (preview && preview.sizeBytes <= 1_000_000) { + const previewBytes = await collectBlob(preview.stream); + const dimensions = sniffImageDimensions(previewBytes); + if (dimensions?.format === previewFormat && dimensions.width <= 512 && dimensions.height <= 512) { + const previewArtifact = await registerArtifact( + register, + "in", + MAX_INBOUND_FILES + metas.length, + name, + previewMimetype, + previewBytes, + true, + ); + previewArtifactId = previewArtifact?.id; + } + } else { + preview?.stream.destroy(); + } + } catch (error) { + preview?.stream.destroy(); + try { + register.onError?.(error); + } catch (reportError) { + swallowAs("attachments: preview onError", undefined)(reportError); + } + } + } metas.push({ name, mimetype, @@ -361,6 +406,7 @@ export async function materializeInbound( ...(a.author ? { author: a.author } : {}), ...(a.sourceId ? { sourceId: a.sourceId } : {}), ...(registered ? { artifactId: registered.id } : {}), + ...(previewArtifactId ? { previewArtifactId } : {}), }); if (VISION_MIME_TYPES.has(mimetype) && bytes.length > 0 && bytes.length <= MAX_VISION_IMAGE_BYTES) { const imageBytes = await downscaleVisionImage(bytes, mimetype); diff --git a/src/files/file-artifact-store.ts b/src/files/file-artifact-store.ts index a2b5972b05..4a0067bc91 100644 --- a/src/files/file-artifact-store.ts +++ b/src/files/file-artifact-store.ts @@ -43,6 +43,7 @@ export interface PutFileInput { createdAt?: number; maxBytes?: number; reuseExistingPath?: boolean; + enabled?: boolean; } type PublishFileInput = Omit & { @@ -78,7 +79,10 @@ export interface FileArtifactStore { get(id: string, opts?: { includeDisabled?: boolean }): Promise; - open(id: string): Promise<{ artifact: FileArtifact; sizeBytes: number; stream: Readable } | null>; + open( + id: string, + opts?: { includeDisabled?: boolean }, + ): Promise<{ artifact: FileArtifact; sizeBytes: number; stream: Readable } | null>; listOwnedByScopes(scopes: readonly ScopeId[], opts?: ListOwnedOptions): Promise; @@ -109,6 +113,14 @@ export function artifactPath(id: string, name: string): string { return `artifacts/${id}/${name}`; } +export function previewArtifactPath(id: string, name: string): string { + return `previews/${id}/${name}`; +} + +export function isPreviewArtifactPath(path: string): boolean { + return /^previews\/[0-9a-f]{32}\//.test(path); +} + export function isArtifactPath(path: string): boolean { return /^artifacts\/[0-9a-f]{32}\//.test(path); } @@ -221,7 +233,7 @@ export function createMemoryFileArtifactStore(byteStore: DurableByteStore): File ...(input.createdInScope ? { createdInScope: input.createdInScope } : {}), createdAt: at, updatedAt: at, - enabled: true, + enabled: input.enabled !== false, }; rows.set(artifact.id, artifact); return { artifact, created: true }; @@ -234,9 +246,9 @@ export function createMemoryFileArtifactStore(byteStore: DurableByteStore): File return r; }, - async open(id) { + async open(id, opts) { const r = rows.get(id); - if (!r || !r.enabled || !r.blobKey) return null; + if (!r || (!r.enabled && !opts?.includeDisabled) || !r.blobKey) return null; const bytes = await byteStore.open(r.blobKey); if (!bytes) return null; return { artifact: r, sizeBytes: bytes.sizeBytes, stream: bytes.stream }; diff --git a/src/files/postgres-file-artifact-store.ts b/src/files/postgres-file-artifact-store.ts index c6e1d811a8..d5266c27ab 100644 --- a/src/files/postgres-file-artifact-store.ts +++ b/src/files/postgres-file-artifact-store.ts @@ -167,7 +167,7 @@ export function createPostgresFileArtifactStore( `INSERT INTO file_artifacts (id, kind, owner_scope_id, path, name, mimetype, size_bytes, blob_key, sha256, direction, created_by, created_in_scope, created_at, updated_at, enabled, source) - VALUES ($1,'file',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$12,TRUE,'live') + VALUES ($1,'file',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$12,$13,'live') ON CONFLICT (id) DO NOTHING`, [ input.id, @@ -182,6 +182,7 @@ export function createPostgresFileArtifactStore( input.createdBy, input.createdInScope ?? null, at, + input.enabled !== false, ], ); const result = await client.query("SELECT * FROM file_artifacts WHERE id=$1", [input.id]); @@ -196,9 +197,9 @@ export function createPostgresFileArtifactStore( return r; }, - async open(id) { + async open(id, opts) { const r = await getRow(id); - if (!r || !r.enabled || !r.blobKey) return null; + if (!r || (!r.enabled && !opts?.includeDisabled) || !r.blobKey) return null; const bytes = await byteStore.open(r.blobKey); if (!bytes) return null; return { artifact: r, sizeBytes: bytes.sizeBytes, stream: bytes.stream }; diff --git a/src/sessions/session-share.ts b/src/sessions/session-share.ts index 65fc82eea3..5c65641c20 100644 --- a/src/sessions/session-share.ts +++ b/src/sessions/session-share.ts @@ -6,6 +6,8 @@ export interface SharedAttachment { name: string; mimetype: string; sizeBytes: number; + inlinePreview?: boolean; + previewId?: string; } export interface SharedMessage { @@ -31,13 +33,29 @@ interface ProjectedMessage { role: "user" | "assistant"; text: string; attachmentIds?: string[]; + inlinePreviewIds?: string[]; + previewPairs?: Array<{ attachmentId: string; previewId: string }>; } -function attachmentIds(value: unknown): string[] { - if (!Array.isArray(value)) return []; - return value.flatMap((file) => - file && typeof file === "object" && typeof file.artifactId === "string" ? [file.artifactId] : [], - ); +function attachmentIds( + value: unknown, + includePreviews = false, +): { ids: string[]; inline: string[]; pairs: Array<{ attachmentId: string; previewId: string }> } { + if (!Array.isArray(value)) return { ids: [], inline: [], pairs: [] }; + const ids: string[] = []; + const inline: string[] = []; + const pairs: Array<{ attachmentId: string; previewId: string }> = []; + for (const item of value) { + if (!item || typeof item !== "object") continue; + const file = item as { artifactId?: unknown; previewArtifactId?: unknown }; + if (typeof file.artifactId !== "string") continue; + ids.push(file.artifactId); + if (includePreviews && typeof file.previewArtifactId === "string") { + inline.push(file.previewArtifactId); + pairs.push({ attachmentId: file.artifactId, previewId: file.previewArtifactId }); + } + } + return { ids, inline, pairs }; } export function sharedMessages( @@ -47,9 +65,21 @@ export function sharedMessages( const messages: ProjectedMessage[] = []; const posts = new Map(); let posted = false; - const emit = (role: "user" | "assistant", text: string, files: string[]) => { + const emit = ( + role: "user" | "assistant", + text: string, + files: string[], + inline: string[] = [], + pairs: Array<{ attachmentId: string; previewId: string }> = [], + ) => { if (!text.trim() && !files.length) return; - messages.push({ role, text, ...(files.length ? { attachmentIds: [...new Set(files)] } : {}) }); + messages.push({ + role, + text, + ...(files.length ? { attachmentIds: [...new Set(files)] } : {}), + ...(inline.length ? { inlinePreviewIds: [...new Set(inline)] } : {}), + ...(pairs.length ? { previewPairs: pairs } : {}), + }); }; for (const entry of entries) { const p = entry.payload; @@ -60,7 +90,8 @@ export function sharedMessages( posts.clear(); if (payload.hidden || payload.overheard) continue; const text = typeof payload.display === "string" && payload.display.trim() ? payload.display : payload.text; - emit("user", typeof text === "string" ? text : "", attachmentIds(payload.attachments)); + const attachments = attachmentIds(payload.attachments, true); + emit("user", typeof text === "string" ? text : "", attachments.ids, attachments.inline, attachments.pairs); } else if (entry.type === "tool_call" && payload.action === "post") { if (typeof payload.callId === "string") posts.set(payload.callId, typeof payload.text === "string" ? payload.text : ""); @@ -70,7 +101,7 @@ export function sharedMessages( posts.delete(callId); if (payload.isError === true || payload.ok === false) continue; if (text !== undefined) { - emit("assistant", text, attachmentIds(payload.files)); + emit("assistant", text, attachmentIds(payload.files).ids); posted = true; } } else if (entry.type === "assistant") { @@ -82,7 +113,7 @@ export function sharedMessages( posted = false; posts.clear(); } else if (entry.type === "delivery") { - const ids = attachmentIds(payload.files); + const ids = attachmentIds(payload.files).ids; const last = messages.at(-1); if (ids.length && last?.role === "assistant") last.attachmentIds = [...new Set([...(last.attachmentIds ?? []), ...ids])]; diff --git a/src/types.ts b/src/types.ts index 0805d271ca..77b53eb0a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -541,6 +541,8 @@ interface BlobAttachment { export type IncomingAttachment = BlobAttachment & { sourceId?: string; author?: string; + previewBlobId?: string; + previewMimetype?: string; }; export type OutgoingAttachment = BlobAttachment & { @@ -555,6 +557,7 @@ export interface AttachmentMeta { direction: "in" | "out"; author?: string; artifactId?: string; + previewArtifactId?: string; sourceId?: string; } diff --git a/test/file-store-population.test.ts b/test/file-store-population.test.ts index bd3243ccf9..a5d9825f33 100644 --- a/test/file-store-population.test.ts +++ b/test/file-store-population.test.ts @@ -10,6 +10,7 @@ import { createAclStore } from "../src/acl/acl-store.ts"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { Readable } from "node:stream"; import { scopeId } from "../src/types.ts"; import type { Sandbox, SandboxHandle } from "../src/sandbox/sandbox.ts"; @@ -17,6 +18,27 @@ const owner = scopeId("channel", "C1"); const HANDLE = { id: "h", rootDir: "/workspace" } as SandboxHandle; const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff, 0xfe, 0x7f]); +function webp(width: number, height: number): Buffer { + const bytes = Buffer.alloc(30); + bytes.write("RIFF", 0, "ascii"); + bytes.writeUInt32LE(22, 4); + bytes.write("WEBP", 8, "ascii"); + bytes.write("VP8X", 12, "ascii"); + bytes.writeUInt32LE(10, 16); + bytes.writeUIntLE(width - 1, 24, 3); + bytes.writeUIntLE(height - 1, 27, 3); + return bytes; +} + +function png(width: number, height: number): Buffer { + const bytes = Buffer.alloc(24); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(bytes); + bytes.write("IHDR", 12, "ascii"); + bytes.writeUInt32BE(width, 16); + bytes.writeUInt32BE(height, 20); + return bytes; +} + function memSandbox(seed: Record = {}): { sandbox: Sandbox; files: Map; @@ -134,6 +156,153 @@ test("materializeInbound registers an 'in' artifact, openable", async () => { ); }); +test("materializeInbound registers a bounded image preview beside the original", async () => { + const store = createMemoryFileArtifactStore(createMemoryDurableByteStore()); + const transfer = createMemoryBlobTransferStore(); + const { blobId } = await transfer.put(PNG); + const preview = webp(512, 384); + const { blobId: previewBlobId } = await transfer.put(preview); + const { sandbox } = memSandbox(); + + const inbound = await materializeInbound( + sandbox, + HANDLE, + [ + { + name: "shared.png", + mimetype: "image/png", + sizeBytes: PNG.length, + blobId, + previewBlobId, + previewMimetype: "image/webp", + }, + ], + transfer, + reg(store), + ); + + assert.notEqual(inbound.metas[0]!.previewArtifactId, inbound.metas[0]!.artifactId); + const opened = await store.open(inbound.metas[0]!.previewArtifactId!, { includeDisabled: true }); + assert.ok(opened); + const chunks: Buffer[] = []; + for await (const chunk of opened.stream) chunks.push(chunk as Buffer); + assert.deepEqual(Buffer.concat(chunks), preview); + assert.equal((await store.listOwnedByScopes([owner])).files.length, 1); +}); + +test("materializeInbound accepts a bounded PNG preview when WebP encoding is unavailable", async () => { + const store = createMemoryFileArtifactStore(createMemoryDurableByteStore()); + const transfer = createMemoryBlobTransferStore(); + const { blobId } = await transfer.put(PNG); + const preview = png(512, 384); + const { blobId: previewBlobId } = await transfer.put(preview); + const { sandbox } = memSandbox(); + + const inbound = await materializeInbound( + sandbox, + HANDLE, + [ + { + name: "shared.png", + mimetype: "image/png", + sizeBytes: PNG.length, + blobId, + previewBlobId, + previewMimetype: "image/png", + }, + ], + transfer, + reg(store), + ); + + const previewArtifactId = inbound.metas[0]!.previewArtifactId; + assert.ok(previewArtifactId); + const opened = await store.open(previewArtifactId, { includeDisabled: true }); + assert.ok(opened); + const chunks: Buffer[] = []; + for await (const chunk of opened.stream) chunks.push(chunk as Buffer); + assert.deepEqual(Buffer.concat(chunks), preview); +}); + +test("materializeInbound rejects preview bytes that do not match their declared MIME type", async () => { + const store = createMemoryFileArtifactStore(createMemoryDurableByteStore()); + const transfer = createMemoryBlobTransferStore(); + const { blobId } = await transfer.put(PNG); + const { blobId: previewBlobId } = await transfer.put(webp(128, 96)); + const { sandbox } = memSandbox(); + + const inbound = await materializeInbound( + sandbox, + HANDLE, + [ + { + name: "shared.png", + mimetype: "image/png", + sizeBytes: PNG.length, + blobId, + previewBlobId, + previewMimetype: "image/png", + }, + ], + transfer, + reg(store), + ); + + assert.equal(inbound.metas[0]!.previewArtifactId, undefined); + assert.equal((await store.listOwnedByScopes([owner])).files.length, 1); +}); + +test("materializeInbound keeps the original when its optional preview stream fails", async () => { + const store = createMemoryFileArtifactStore(createMemoryDurableByteStore()); + const transfer = createMemoryBlobTransferStore(); + const { blobId } = await transfer.put(PNG); + const { blobId: previewBlobId } = await transfer.put(webp(64, 64)); + const errors: unknown[] = []; + const failingTransfer = { + ...transfer, + open: async (id: string) => + id === previewBlobId + ? { + sizeBytes: 30, + stream: new Readable({ + read() { + this.destroy(new Error("preview read failed")); + }, + }), + } + : transfer.open(id), + }; + const { sandbox } = memSandbox(); + + const inbound = await materializeInbound( + sandbox, + HANDLE, + [ + { + name: "shared.png", + mimetype: "image/png", + sizeBytes: PNG.length, + blobId, + previewBlobId, + previewMimetype: "image/webp", + }, + ], + failingTransfer, + reg(store, { + onError: (error) => { + errors.push(error); + throw new Error("preview error report failed"); + }, + }), + ); + + assert.ok(inbound.metas[0]!.artifactId); + assert.equal(inbound.metas[0]!.previewArtifactId, undefined); + assert.equal((await store.listOwnedByScopes([owner])).files.length, 1); + assert.deepEqual(await drain(store, inbound.metas[0]!.artifactId!), PNG); + assert.equal(errors.length, 1); +}); + test("materializeInbound omits artifactId when registration fails or is absent", async () => { const transfer = createMemoryBlobTransferStore(); const { blobId } = await transfer.put(PNG); diff --git a/test/session-sharing.test.ts b/test/session-sharing.test.ts index 7389f512ad..b10a7490d7 100644 --- a/test/session-sharing.test.ts +++ b/test/session-sharing.test.ts @@ -96,6 +96,8 @@ test("fresh shares freeze messages and authorized attachments with separate audi let visible = original; let accessible = true; let fileData: string | null = ""; + let fileName = "example.html"; + let fileMime = "text/html"; const server = createServer(async (req, res) => { const url = new URL(req.url!, "http://localhost"); const parts = url.pathname.split("/"); @@ -135,15 +137,17 @@ test("fresh shares freeze messages and authorized attachments with separate audi entries: visible, } : null, - openFileForViewer: async (id: string, user: string) => - id === "f1" && user === "alice" && fileData !== null - ? { - name: "example.html", - mimetype: "text/html", - sizeBytes: Buffer.byteLength(fileData), - stream: Readable.from(fileData), - } - : null, + openFileForViewer: async (id: string, user: string, opts?: { preview?: boolean }) => { + if (user !== "alice" || fileData === null || !["f1", "p1"].includes(id)) return null; + if (id === "p1" && !opts?.preview) return null; + const data = id === "p1" ? "PREVIEW" : fileData; + return { + name: id === "p1" ? "example.webp" : fileName, + mimetype: id === "p1" ? "image/webp" : fileMime, + sizeBytes: Buffer.byteLength(data), + stream: Readable.from(data), + }; + }, }, } as unknown as ApiCtx); }); @@ -181,6 +185,29 @@ test("fresh shares freeze messages and authorized attachments with separate audi assert.equal((await read(external.share.token, "external", "", "")).status, 200); assert.equal((await read(external.share.token)).status, 404); assert.equal((await read(second.share.token, "internal", `/files/${fileId}`)).status, 404); + fileData = "BM"; + fileName = "example.bmp"; + fileMime = "image/bmp"; + visible = [ + ...entries, + entry("user", { text: "Image", attachments: [{ artifactId: "f1", previewArtifactId: "p1" }] }, 12), + ]; + const imageShare = (await (await create()).json()) as { share: { token: string } }; + const imageSnapshot = (await (await read(imageShare.share.token)).json()) as { + messages: Array<{ attachments?: Array<{ id: string; previewId?: string }> }>; + }; + const imageAttachment = imageSnapshot.messages.find((message) => message.attachments?.length)?.attachments?.[0]; + const imageFileId = imageAttachment?.id; + assert.ok(imageFileId); + assert.ok(imageAttachment.previewId); + const inlineImage = await read(imageShare.share.token, "internal", `/files/${imageFileId}`); + assert.equal(inlineImage.headers.get("content-type"), "image/bmp"); + assert.match(inlineImage.headers.get("content-disposition")!, /^inline;/); + const previewImage = await read(imageShare.share.token, "internal", `/files/${imageAttachment.previewId}`); + assert.equal(await previewImage.text(), "PREVIEW"); + fileData = ""; + fileName = "example.html"; + fileMime = "text/html"; visible = [entry("user", { text: "Generate file" }, 1), entry("assistant", { text: "Generated" }, 2)]; const delivery = await deliveries.enqueue({ destination: { type: "web", target: "web:alice:s1" }, @@ -248,7 +275,7 @@ test("fresh shares freeze messages and authorized attachments with separate audi test("attachment projection includes only user and delivered attachments", () => { const messages = sharedMessages([ - entry("user", { attachments: [{ artifactId: "user" }] }, 1), + entry("user", { attachments: [{ artifactId: "user", previewArtifactId: "user-preview" }] }, 1), entry("tool_result", { tool: "execute", files: [{ artifactId: "private" }] }, 2), entry("tool_result", { tool: "attach", files: [{ artifactId: "failed" }], isError: true }, 3), entry("tool_result", { tool: "attach", files: [{ artifactId: "old", name: "a" }] }, 4), @@ -260,7 +287,13 @@ test("attachment projection includes only user and delivered attachments", () => entry("assistant", { text: "private final" }, 10), ]); assert.deepEqual(messages, [ - { role: "user", text: "", attachmentIds: ["user"] }, + { + role: "user", + text: "", + attachmentIds: ["user"], + inlinePreviewIds: ["user-preview"], + previewPairs: [{ attachmentId: "user", previewId: "user-preview" }], + }, { role: "assistant", text: "Here", attachmentIds: ["new"] }, { role: "assistant", text: "Posted", attachmentIds: ["posted"] }, ]);