diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d4467a7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: ci + +on: + pull_request: + push: + branches: [main] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm -r typecheck + # Only the network-free tests run in CI; the voice/room/desktop-window + # suites need live APIs, tmux, the kanban CLI or a Mac (see README). The + # room unit tests (project resolution, channel naming, the agent brief) + # and the desktop tiler/noop tests are pure. + - run: npx vitest run tests/room.unit.test.ts tests/intent.unit.test.ts tests/desktop.test.ts + working-directory: server diff --git a/DEMO.md b/DEMO.md new file mode 100644 index 0000000..5d564d7 --- /dev/null +++ b/DEMO.md @@ -0,0 +1,155 @@ +# Demo runbook + +## Setup (2 minutes, do this before people walk in) + +```bash +# the site the agents work on (keep it running) +cd ~/Projects/2lang-2watch && pnpm dev # http://127.0.0.1:5173 + +cd ~/Projects/tokenmaxxer +pnpm dev # gateway :4870 + console :5170 +pnpm hud # the floating always-on-top listening HUD (Electron) +``` + +Have the **KanbanCode** app open β€” it's the board that shows the rooms and +the agents chatting. Start the vite dev servers FRESH right before the demo +(long-running ones can wedge their tailwind pipeline). + +Grant **Accessibility** to the terminal/Electron once (System Settings β†’ +Privacy & Security β†’ Accessibility) so window tiling works, and +**Microphone** on first listen. Click **listen** on the HUD; Max greets the +room. You're live. + +**The desktop is the demo.** A spoken mission spins up real Claude agents in +real Warp terminals, the KanbanCode board comes forward on the room's +channel so you watch them coordinate, and `open_url` puts the live site or a +GitHub issue on the screen. Set `TOKENMAXXER_DESKTOP=0` to run headless (the +console at http://localhost:5170 still shows the transcript, tool feed, and +live fleet). + +Pre-flight checks (one command, every moving part): + +```bash +cd server && npx tsx scripts/preflight.ts # πŸš€ ALL SYSTEMS GO or it tells you what's broken +``` + +Then the human check: green **LIVE** dot, say "Max, quick check, you there?" +β†’ Max answers within ~2s. + +If gemma-4 reports capacity problems the gateway retries and then switches +to the fallback model automatically β€” the demo keeps going (a banner shows +the switch). + +## The 2-minute skit (the exact wording we say) + +Two devs in a standup β€” **R** (Rogerio) and **A** (Aryan) β€” talking to each +other. **M** is Max, the always-listening room AI. Nobody touches a keyboard. +Screen starts empty: KanbanCode board idle top-left, HUD pulsing bottom-centre. + +> **A:** "Did the new site go live this morning?" +> **R:** "Think so β€” Max, pull it up?" +> **M:** *(site flies onto the screen, top-right)* "There it is." +> **A:** "Ohh, that's clean." +> **R:** "Clean, but it's blinding. We're devs β€” we live in dark mode." +> **A:** "Honestly it'd look way cooler dark. Max, get a few agents on a dark-mode pass." +> **M:** "Say less, boss β€” three agents on full dark mode, they're in the room now." *(3 terminals fly open; KanbanCode board snaps to #dark-mode top-left)* +> **A:** *(reading the board)* "Wait β€” one of them just said 'I'll take the CSS foundations, someone grab the pages.'" +> **R:** "We didn't assign that." +> **M:** "They sort it out themselves. That's the whole point." +> +> *(let it land β€” you assigned no roles)* +> +> **A:** "Oh β€” someone opened issue 1234 on the repo overnight. Max, throw it on the screen?" +> **M:** *(GitHub issue, top-right)* "Up." +> **R:** "Don't wait on us β€” get a couple agents fixing it too." +> **M:** "Second room, going." *(more terminals; two rooms running in parallel now)* +> **R:** *(to the room)* "We haven't typed one character. We just talked." +> **M:** "Five agents shipping. Six now β€” the dark-mode room pulled in another." + +**Tag line:** *"When compute isn't the limit, you don't write code β€” you run a meeting."* + +**Why it works as a story** +- One line of small talk, max β€” 2 minutes is brutal; open inside the action. +- The hero beat is the **self-organization** ("I'll take the CSS, you grab the + pages" β€” unprompted). Read it aloud off the board and pause. That's the line + that makes judges sit up: *you assigned no roles.* +- Give Max a voice β€” dry, fast, a little cocky ("Say less", "Six now"). One + memorable line per beat. +- Choreograph the screen so it FILLS UP: site β†’ terminals + board β†’ issue β†’ + second room. The accumulation is "unlimited compute" made visible; never + clear anything. +- The second room (the issue) proves it **scales** β€” one meeting, two parallel + workstreams. End on the reframe and stop talking. + +**Reliability (so the magic lands live)** β€” these exact cues route 100% in the +delegation eval: "pull it up" / "pull up the website" -> open_url; "get a few +agents on dark mode" / "get a couple agents fixing it" -> spawn_room; "throw +issue 1234 on the screen" -> open_url. Keep an action verb in the human's line +(or let Max carry it). A pure vibe line with no verb ("it'd look cooler dark") +may not trigger Max on its own. + +## The arc (the swarm is the story) + +1. **The hook.** "This room has unlimited compute. One agent is one context + window on one thread. Watch what ten in parallel looks like." Small talk + with Max β€” show it's a real conversation, sub-second replies. + +2. **Put the work on the screen.** "Max, pull up the new website." A real + browser window flies open at the running site (`open_url` resolves "the + website" to the local site β€” no port guessing). + +3. **Spin up a room (the Iron Man moment).** "Get five agents on a full dark + mode for the website." Five Warp terminals fly open with Claude Code, the + KanbanCode board comes forward on the `#full-dark-mode` channel, and you + watch the agents self-organize: one says "I'll take the CSS foundations + and global components, someone take the pages", another claims the pages, + a third reviews. **You assigned no roles.** That's the agent loop β€” a + natural ralph-loop where the agents hold each other to the mission. + +4. **Steer by voice, live.** Keep talking to the room: "tell them to keep the + logo readable on black" (`message_room`), "throw two more agents at it" + (`add_agents`). The team reacts in the channel without you touching a + keyboard. + +5. **A second front.** "We saw someone opened issue 1234 on langwatch β€” pull + it up." The issue opens on the screen (`open_url`). "Get a couple agents + to fix it." A new room spins up on the langwatch repo, its own channel, + its own terminals. Two missions running in parallel. + +6. **The kicker (proactive).** Keep talking about something else. When a room + lands a PR, a notification fires and the PR window pops on its own β€” you + didn't ask. Or: "Max, how's it going in there?" and he reads the room + back to you from the channel (`check_progress`). + +The pitch in one line: separate, visible sessions give you full control of N +context windows working in parallel β€” not one agent burning tokens on hidden +sub-tasks. + +## Things that can go sideways + +| symptom | fix | +| --- | --- | +| Max silent after a question | semantic VAD missed the turn end β€” say a clear short sentence; barge-in always works | +| "model at capacity" banner | automatic β€” retries then falls back; keep talking | +| Max opens a browser instead of spinning a room | he misheard "build" as "show" β€” rephrase as "get agents on X" / "spin up a room to X" | +| agents launched but not talking yet | claude takes a few seconds to boot; they join the channel once ready β€” watch the board | +| board didn't come forward | grant Accessibility; or click the KanbanCode channel yourself (`kanban channel open `) | +| terminals open but don't tile | grant Accessibility to the terminal/Electron; they still open, just not snapped | +| browser window on the wrong Space | it shouldn't β€” we launch a dedicated chromeless instance on the current Space | +| echo / Max hears itself | echo cancellation is on; lower speaker volume a notch | + +## Reset between runs + +```bash +# stop the agents, leave their cards +tmux ls | grep '^tmx-' | cut -d: -f1 | xargs -n1 tmux kill-session -t +# reset the demo site +git -C ~/Projects/2lang-2watch checkout . && git -C ~/Projects/2lang-2watch clean -fd +# clear the room channels + their history +for c in $(ls ~/.kanban-code/channels/*.jsonl 2>/dev/null | xargs -n1 basename | sed 's/.jsonl//'); do + case "$c" in dark-mode|full-dark-mode|login*|issue*) kanban channel delete "$c"; rm -f ~/.kanban-code/channels/$c.jsonl;; esac +done +# close prototype/browser windows +pkill -f tokenmaxxer-chrome +rm -f ~/.warp/launch_configurations/tokenmaxxer-*.yaml +``` diff --git a/README.md b/README.md index 385ae7f..f520d88 100644 --- a/README.md +++ b/README.md @@ -4,49 +4,106 @@ Hackathon theme: *what if compute and tokens were no longer a limitation?* -You sit down for a day of work. Microphones are on. Max β€” a realtime -voice agent β€” is in the room with you. You exchange ideas, argue, change -your mind. Behind your voice, Max is spinning up Claude Code agents by the -dozen, reusing the ones already on the right workstream, and painting -prototype pages on the room screen at 17,000 tokens per second. Nothing -waits for the meeting to end. Some of the work gets thrown away. Compute -is unlimited β€” hesitation is the only cost. +You sit down for a day of work. Microphones are on. Max, a realtime voice +agent, is in the room with you. You say "get five agents on a full dark mode +for the website" and five Claude Code agents spin up, pile into one shared +chat channel, and self-organize, one claims the CSS foundations, another +takes the pages, a third reviews. You didn't assign roles. You keep talking; +they keep working. Compute is unlimited, hesitation is the only cost. -## Architecture +## It controls your machine, not a tab + +tokenmaxxer is not a webpage you build inside, it drives your real desktop. +Say "spin up a room to fix the login" and real terminals fly open with Claude +Code agents working in them. The KanbanCode board comes forward showing the +agents talk to each other in real time. Say "pull up the website" and a real +browser window opens at the running site. A floating HUD shows the room is +listening, on top of everything. ``` - room mic ──► console (browser) ──ws──► gateway (server) ──ws──► Inworld Realtime - β–² β–² β”‚ tool calls (STT+LLM+TTS, gemma-4) - β”‚ β”‚ β”œβ”€β–Ί write_page / edit_page ──► chatjimmy (~17k tok/s) - playground β—„β”€β”€β”€β”˜ β”‚ β”‚ └──► playground/src/pages/*.tsx (vite HMR) - preview β”‚ └─► dispatch_work / check_progress - β”‚ └──► orchestrator (Claude brain) - fleet + tool feed └──► kanban CLI ──► tmux ──► N Γ— claude code + room mic ──► HUD overlay (Electron) ──ws──► gateway (server, on your Mac) + always-on-top, transparent β”‚ ←wsβ†’ Inworld Realtime (gemma-4) + β”‚ + β”Œβ”€β”€ desktop control (osascript / Warp) ◄───────── tool calls + β”‚ β€’ spawn_room β†’ N Warp terminals, tmux, Claude Code, watch them work + β”‚ β€’ focusChannel β†’ KanbanCode board forward on the room's channel + β”‚ β€’ open_url β†’ chromeless Chrome window at the live site / a GitHub issue + β”‚ β€’ done β†’ macOS notification + the PR window, proactively + β”‚ + └── the swarm appears AROUND the HUD as real windows + β”‚ + room ◄── kanban channel (the agent loop) ── kanban CLI ── tmux ── N Γ— claude code + agents claim slices, review each other, coordinate ``` -- **server/** β€” voice gateway (OpenAI Realtime-compatible WS endpoint, proxies - Inworld), tool execution, the orchestrator brain, the jimmy speed-codegen - client, scenario voice tests. -- **console/** β€” the room screen: mic/audio, live transcript, tool activity - feed, agent fleet panel, embedded playground preview. -- **playground/** β€” pre-scaffolded vite + react + tailwind site with - file-per-page routing. Written to by jimmy in seconds, deepened by Claude - agents in minutes. +### Why this wins (the swarm, not a single hidden agent) + +- **Agentic leverage** β€” N independent Claude agents on one mission, plus a + voice agent listening live to steer them. Not one agent with hidden + sub-tasks: N real sessions you can see and talk to. +- **Measurable impact** β€” 1 agent is one context window on one thread. N + agents is N context windows working the problem in parallel. Separate + sessions give you full control of all of them. +- **The agent loop** β€” the shared channel is a natural ralph-loop: agents + keep messaging each other, finding issues, and pushing forward. One can + step up as supervisor. No better loop than agents holding each other to it. +- **Cost / quality** β€” because the agents are under your control and visible, + they don't burn tokens mysteriously doing random things. +- **Collaboration** β€” the team guides the swarm by voice, live, together. +- **Technical ambition** β€” the desktop layer is built behind a + platform-agnostic adapter so it runs on Windows too, not just macOS. + +## The pieces + +- **server/** β€” voice gateway (OpenAI Realtime-compatible WS, proxies + Inworld), server-side tool execution, the **room engine** + (`orchestrator/room.ts`: spawn a swarm into a kanban channel, the + coordinate-first agent brief, a deterministic project registry), and the + **desktop control layer** (platform-agnostic `DesktopController`; macOS + adapter via osascript + Warp + chromeless Chrome; Windows stub for the + incoming teammate). +- **hud/** β€” Electron shell: a frameless, transparent, always-on-top overlay + wrapping the console in HUD mode (reuses the console's mic/audio/WS stack). +- **console/** β€” the room console (transcript, tool feed, live fleet) and the + compact HUD view (`?hud=1`). - **specs/** β€” BDD feature specs for everything above. +The agents talk over [Kanban Code](https://github.com/langwatch) channels: +each agent has a tmux session kanban tracks, and a `kanban channel send` +broadcasts into every other agent's pane. Max joins the channel too (as +`@max`) so he can drop the mission in and pass along corrections by voice. + ## Run it ```bash pnpm install -cp .env.example .env # fill in the keys -pnpm dev # gateway + console + playground +cp .env.example .env # INWORLD_API_KEY, ANTHROPIC_API_KEY, OPENAI_API_KEY, + # LANGWATCH_API_KEY +pnpm dev # gateway :4870 + console :5170 +pnpm hud # the floating always-on-top listening HUD (Electron) ``` -Open the console, click **start listening**, and talk. +Desktop control is on by default on macOS (`TOKENMAXXER_DESKTOP=0` to run +headless). Click **listen** on the HUD and talk. Demo script and failure +playbook: [DEMO.md](DEMO.md). + +The agents spawn through the kanban CLI in tmux; `TOKENMAXXER_AGENT_MODEL` +picks their model (`sonnet` for testing). The project a room works in is +resolved from what you say ("the website" β†’ the local site checkout, +"langwatch" β†’ the langwatch repo) in `server/src/orchestrator/projects.ts`. ## Test it ```bash -pnpm test # integration + scenario voice tests (real audio, no mocks) +cd server +npx vitest run tests/room.unit.test.ts tests/desktop.test.ts # network-free, CI pnpm typecheck ``` + +The network-free unit tests cover the engine's pure core: how a spoken hint +resolves to a project, how a topic becomes a channel name, and the +coordinate-first brief every agent receives (it must tell the agent to JOIN +the channel and coordinate BEFORE building, that ordering is the whole +delegation contract). The scenario voice tests and the delegation experiment +eval drive the real gateway with audio and a judge; they need live APIs, the +kanban CLI, and tmux, so they run locally, not in CI. diff --git a/console/index.html b/console/index.html new file mode 100644 index 0000000..50da019 --- /dev/null +++ b/console/index.html @@ -0,0 +1,12 @@ + + + + + + tokenmaxxer β€” the room is listening + + +
+ + + diff --git a/console/package.json b/console/package.json new file mode 100644 index 0000000..bab936a --- /dev/null +++ b/console/package.json @@ -0,0 +1,25 @@ +{ + "name": "@tokenmaxxer/console", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite --port 5170 --strictPort", + "build": "vite build", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "echo 'covered by scenario voice tests against the gateway'" + }, + "dependencies": { + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.0", + "@types/react": "^19.2.0", + "@types/react-dom": "^19.2.0", + "@vitejs/plugin-react": "^5.0.0", + "tailwindcss": "^4.1.0", + "typescript": "^5.9.0", + "vite": "^7.1.0" + } +} diff --git a/console/public/worklets.js b/console/public/worklets.js new file mode 100644 index 0000000..c84476a --- /dev/null +++ b/console/public/worklets.js @@ -0,0 +1,82 @@ +/** + * Audio worklets for the meeting console. The AudioContext runs at 24kHz + * to match the Inworld wire format, so no resampling is needed anywhere. + */ + +/** Batches mic input into ~40ms Float32 chunks posted to the main thread. */ +class CaptureProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.buffer = new Float32Array(960); // 40ms @ 24kHz + this.offset = 0; + } + + process(inputs) { + const channel = inputs[0]?.[0]; + if (!channel) return true; + let i = 0; + while (i < channel.length) { + const n = Math.min(channel.length - i, this.buffer.length - this.offset); + this.buffer.set(channel.subarray(i, i + n), this.offset); + this.offset += n; + i += n; + if (this.offset === this.buffer.length) { + this.port.postMessage(this.buffer.slice(0)); + this.offset = 0; + } + } + return true; + } +} + +/** + * Plays PCM16 chunks from a growing queue. "flush" empties it instantly + * (barge-in). Posts {playing} transitions so the UI can show talk state. + */ +class PlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.current = null; + this.offset = 0; + this.wasPlaying = false; + this.port.onmessage = (e) => { + if (e.data === "flush") { + this.queue = []; + this.current = null; + this.offset = 0; + } else { + this.queue.push(e.data); + } + }; + } + + process(_inputs, outputs) { + const out = outputs[0][0]; + let filled = 0; + while (filled < out.length) { + if (!this.current || this.offset >= this.current.length) { + this.current = this.queue.shift() ?? null; + this.offset = 0; + if (!this.current) break; + } + const n = Math.min(out.length - filled, this.current.length - this.offset); + for (let i = 0; i < n; i++) { + out[filled + i] = this.current[this.offset + i] / 32768; + } + filled += n; + this.offset += n; + } + for (let i = filled; i < out.length; i++) out[i] = 0; + + const playing = filled > 0; + if (playing !== this.wasPlaying) { + this.wasPlaying = playing; + this.port.postMessage({ playing }); + } + return true; + } +} + +registerProcessor("tokenmaxxer-capture", CaptureProcessor); +registerProcessor("tokenmaxxer-playback", PlaybackProcessor); diff --git a/console/src/App.tsx b/console/src/App.tsx new file mode 100644 index 0000000..84a85ba --- /dev/null +++ b/console/src/App.tsx @@ -0,0 +1,244 @@ +import { useEffect, useState } from "react"; +import { room, type RoomState } from "./realtime"; + +const PLAYGROUND_URL = `http://${window.location.hostname}:5171`; + +function useRoom(): RoomState { + const [state, setState] = useState(room.state); + useEffect(() => room.subscribe(setState), []); + return state; +} + +function ConnectButton({ state }: { state: RoomState }) { + if (state.connection === "idle") { + return ( + + ); + } + const label = { + connecting: "connecting…", + live: "live", + reconnecting: "reconnecting…", + }[state.connection]; + return ( +
+ + + {state.agentTalking ? "max is talking" : label} + + +
+ ); +} + +function Transcript({ state }: { state: RoomState }) { + return ( +
+

+ conversation +

+
+ {[...state.transcript].reverse().map((t, i) => ( +
+ {t.text} +
+ ))} + {state.transcript.length === 0 && ( +

+ Start talking β€” everything lands here. +

+ )} +
+
+ ); +} + +const TOOL_ICONS: Record = { + dispatch_work: "πŸš€", + check_progress: "πŸ“‘", + write_page: "🎨", + edit_page: "✏️", + open_page: "πŸ–₯️", +}; + +function ToolFeed({ state }: { state: RoomState }) { + return ( +
+

+ actions +

+
+ {state.tools.map((t) => ( +
+
+ {TOOL_ICONS[t.tool] ?? "βš™οΈ"} + {t.tool} + + {t.phase === "started" + ? "running…" + : t.elapsedMs !== undefined + ? `${(t.elapsedMs / 1000).toFixed(1)}s` + : ""} + +
+
+ {summarizeArgs(t.args)} +
+ {t.result && t.phase !== "started" && ( +
{t.result}
+ )} +
+ ))} + {state.tools.length === 0 && ( +

+ Tool calls show up here the moment Max fires them. +

+ )} +
+
+ ); +} + +function summarizeArgs(args: Record): string { + return Object.entries(args) + .map(([k, v]) => `${k}: ${String(v)}`) + .join(" Β· "); +} + +const STATUS_STYLE: Record = { + launching: "bg-amber-400", + working: "bg-emerald-400 talking", + idle: "bg-sky-400", + gone: "bg-zinc-600", +}; + +function FleetPanel({ state }: { state: RoomState }) { + return ( +
+

+ the fleet{" "} + ({state.fleet.length} agents) +

+
+ {state.fleet.map((a) => ( +
+
+ + {a.slug} + + {a.status} + +
+
{a.topic}
+
+ {a.lastActivity} +
+
+ ))} + {state.fleet.length === 0 && ( +

+ No agents yet. Mention work and watch them spawn. +

+ )} +
+
+ ); +} + +function Preview({ state }: { state: RoomState }) { + return ( +
+
+

+ room screen +

+ + {state.previewPath} + +
+