Render conversation images inline

plugins/chassis/src/image-mime.ts

@@ -0,0 +1,18 @@
+const RENDERABLE_IMAGE_TYPES = new Set([
+  "image/png",
+  "image/jpeg",
+  "image/jpg",
+  "image/pjpeg",
+  "image/gif",
+  "image/webp",
+  "image/avif",
+  "image/bmp",
+  "image/apng",
+  "image/x-icon",
+  "image/vnd.microsoft.icon",
+  "image/x-ms-bmp",
+]);
+
+export function browserRenderableImage(mimeType?: string): boolean {
+  return RENDERABLE_IMAGE_TYPES.has((mimeType ?? "").split(";")[0]!.trim().toLowerCase());
+}

plugins/web-ui/server/index.ts

@@ -789,6 +789,8 @@ interface CoreAttachment {
   mimetype: string;
   sizeBytes: number;
   blobId: string;
+  previewBlobId?: string;
+  previewMimetype?: string;
 }
 
 interface CoreApprovalRecord {
@@ -1038,7 +1040,7 @@ async function serveFileContent(c: WebCtx, playground = false): Promise<unknown>
   const { res, user, url } = c;
   const id = c.params.id!;
   const corePath = withSourceAuthNonce(
-    `/v1/files/${encodeURIComponent(id)}/content?viewer=${encodeURIComponent(user)}`,
+    `/v1/files/${encodeURIComponent(id)}/content?viewer=${encodeURIComponent(user)}${url.searchParams.get("preview") === "1" ? "&preview=1" : ""}`,
     CORE_SIGNING_SECRET,
   );
   const portalTok = portalTokenStore.getStore();
@@ -2330,13 +2332,27 @@ const apiRoutes: readonly WebRoute[] = [
         if (Array.isArray(p.attachments)) {
           for (const raw of p.attachments as unknown[]) {
             if (!raw || typeof raw !== "object") continue;
-            const a = raw as { name?: unknown; mimetype?: unknown; sizeBytes?: unknown; blobId?: unknown };
+            const a = raw as {
+              name?: unknown;
+              mimetype?: unknown;
+              sizeBytes?: unknown;
+              blobId?: unknown;
+              previewBlobId?: unknown;
+              previewMimetype?: unknown;
+            };
             if (typeof a.name !== "string" || typeof a.blobId !== "string" || !a.blobId) continue;
             attachments.push({
               name: a.name,
               mimetype: typeof a.mimetype === "string" && a.mimetype ? a.mimetype : "application/octet-stream",
               sizeBytes: typeof a.sizeBytes === "number" ? a.sizeBytes : 0,
               blobId: a.blobId,
+              ...(typeof a.previewBlobId === "string" && a.previewBlobId
+                ? {
+                    previewBlobId: a.previewBlobId,
+                    previewMimetype:
+                      typeof a.previewMimetype === "string" ? a.previewMimetype : "application/octet-stream",
+                  }
+                : {}),
             });
           }
         }

plugins/web-ui/src/chat.ts

@@ -686,6 +686,8 @@ export function createChatSurface(
 
   function dispose(): void {
     redrawHooks.delete(redrawForConnector);
+    for (const url of localAttachmentUrls.values()) URL.revokeObjectURL(url);
+    localAttachmentUrls.clear();
     teardownActiveChat();
   }
 
@@ -1630,7 +1632,13 @@ export function createChatSurface(
               ${edited || deleted ? html`<span class="revision-badge">(${deleted ? "deleted" : "edited"})</span>` : nothing}
             </div>
             <button class="pin-toggle" type="button" hidden aria-expanded="false">Show more</button>
-            ${attachments.length ? html`<div class="message-files">${attachments.map(userAttachmentBadge)}</div>` : nothing}
+            ${
+              attachments.length
+                ? html`<div class="message-files">
+                    ${attachments.map((attachment) => userAttachmentBadge(attachment, message))}
+                  </div>`
+                : nothing
+            }
           </div>
           ${
             sendFailure
@@ -2797,10 +2805,13 @@ export function createChatSurface(
     mimeType?: string;
     size?: number;
     content?: string;
+    preview?: string;
     artifactId?: string;
+    previewArtifactId?: string;
   }
 
   const localAttachmentUrls = new Map<UserAttachmentView, string>();
+  const brokenUserImageSources = new Set<string>();
 
   function localContentUrl(a: UserAttachmentView): string | undefined {
     if (!a.content) return undefined;
@@ -2818,13 +2829,28 @@ export function createChatSurface(
     }
   }
 
-  function userAttachmentBadge(a: UserAttachmentView): TemplateResult {
+  function userAttachmentBadge(a: UserAttachmentView, message: AgentMessage): TemplateResult | typeof nothing {
     const artifactHref = a.artifactId ? fileContentUrl(a.artifactId, a.fileName) : undefined;
     if (a.mimeType?.startsWith("image/")) {
-      const dataUrl =
-        a.content && (a.content.startsWith("data:") ? a.content : `data:${a.mimeType};base64,${a.content}`);
-      const download = !artifactHref || !browserRenderableImage(a.mimeType);
-      return chipBadge(FileImage, a.fileName, a.size, artifactHref ?? dataUrl ?? undefined, download);
+      const preview = a.preview?.startsWith("data:image/") ? a.preview : undefined;
+      const persistedPreview = a.previewArtifactId
+        ? `${fileContentUrl(a.previewArtifactId, a.fileName)}?preview=1`
+        : undefined;
+      const src = preview ?? persistedPreview;
+      if (browserRenderableImage(a.mimeType) && src && !brokenUserImageSources.has(src)) {
+        return html`<img
+          class="user-image-attachment"
+          src=${src}
+          alt="Attached image"
+          loading="lazy"
+          @error=${() => {
+            brokenUserImageSources.add(src);
+            settledRowCache.delete(message as object);
+            redrawTranscript();
+          }}
+        />`;
+      }
+      return imageChip(a.fileName, a.size, artifactHref);
     }
     if (inlineHtmlName(a.fileName, a.mimeType)) {
       let src = artifactHref;