diff --git a/packages/app/src/components/dialog-connect-provider.tsx b/packages/app/src/components/dialog-connect-provider.tsx index 90f4f41f7c6..c7f9dc8582d 100644 --- a/packages/app/src/components/dialog-connect-provider.tsx +++ b/packages/app/src/components/dialog-connect-provider.tsx @@ -309,6 +309,68 @@ export function DialogConnectProvider(props: { provider: string }) { ) } + function LiteLLMAuthView() { + const [formStore, setFormStore] = createStore({ + baseURL: "", + apiKey: "", + }) + + async function handleSubmit(e: SubmitEvent) { + e.preventDefault() + + const data = new FormData(e.currentTarget as HTMLFormElement) + const url = (data.get("baseURL") as string)?.trim() || "http://localhost:4000" + const key = (data.get("apiKey") as string)?.trim() + + await globalSDK.client.global.config.update({ + config: { + provider: { + litellm: { + options: { baseURL: url }, + }, + }, + }, + }) + if (key) { + await globalSDK.client.auth.set({ + providerID: props.provider, + auth: { type: "api", key }, + }) + } + await complete() + } + + return ( +
+
+ Connect to a LiteLLM proxy server. Models will be discovered automatically. +
+
+ setFormStore("baseURL", v)} + /> + setFormStore("apiKey", v)} + /> + + +
+ ) + } + function OAuthCodeView() { const [formStore, setFormStore] = createStore({ value: "", @@ -479,6 +541,9 @@ export function DialogConnectProvider(props: { provider: string }) { + + + diff --git a/packages/opencode/src/bun/registry.ts b/packages/opencode/src/bun/registry.ts index c567668acd7..e7cbcd2729c 100644 --- a/packages/opencode/src/bun/registry.ts +++ b/packages/opencode/src/bun/registry.ts @@ -19,7 +19,20 @@ export namespace PackageRegistry { }, }) - const code = await result.exited + const code = await Promise.race([ + result.exited, + new Promise((resolve) => setTimeout(() => resolve(null), 10_000)), + ]) + + if (code === null) { + result.kill() + result.stdout?.cancel().catch(() => {}) + result.stderr?.cancel().catch(() => {}) + result.unref() + log.warn("bun info timed out", { pkg, field }) + return null + } + const stdout = result.stdout ? await readableStreamToText(result.stdout) : "" const stderr = result.stderr ? await readableStreamToText(result.stderr) : "" diff --git a/packages/opencode/src/cli/cmd/tui/component/dialog-provider.tsx b/packages/opencode/src/cli/cmd/tui/component/dialog-provider.tsx index 9682bee4ead..3a9bb38f473 100644 --- a/packages/opencode/src/cli/cmd/tui/component/dialog-provider.tsx +++ b/packages/opencode/src/cli/cmd/tui/component/dialog-provider.tsx @@ -83,6 +83,9 @@ export function createDialogProviderOptions() { } } if (method.type === "api") { + if (provider.id === "litellm") { + return dialog.replace(() => ) + } return dialog.replace(() => ) } }, @@ -241,3 +244,52 @@ function ApiMethod(props: ApiMethodProps) { /> ) } + +function LiteLLMMethod() { + const dialog = useDialog() + const sdk = useSDK() + const sync = useSync() + const { theme } = useTheme() + + return ( + ( + Enter the base URL of your LiteLLM proxy server. + )} + onConfirm={async (baseURL) => { + const url = baseURL?.trim() || "http://localhost:4000" + dialog.replace(() => ( + ( + Enter the API key for your LiteLLM proxy, or leave empty if not required. + )} + onConfirm={async (apiKey) => { + await sdk.client.global.config.update({ + config: { + provider: { + litellm: { + options: { baseURL: url }, + }, + }, + }, + }) + if (apiKey?.trim()) { + await sdk.client.auth.set({ + providerID: "litellm", + auth: { type: "api", key: apiKey.trim() }, + }) + } + await sdk.client.instance.dispose() + await sync.bootstrap() + dialog.replace(() => ) + }} + /> + )) + }} + /> + ) +} diff --git a/packages/opencode/src/provider/litellm.ts b/packages/opencode/src/provider/litellm.ts new file mode 100644 index 00000000000..5f54071b500 --- /dev/null +++ b/packages/opencode/src/provider/litellm.ts @@ -0,0 +1,246 @@ +import { Log } from "../util/log" +import { Env } from "../env" +import type { Provider } from "./provider" + +export namespace LiteLLM { + const log = Log.create({ service: "litellm" }) + + interface ModelInfoEntry { + model_name: string + litellm_params?: { + model?: string + [key: string]: unknown + } + model_info?: { + id?: string + input_cost_per_token?: number | null + output_cost_per_token?: number | null + cache_read_input_token_cost?: number | null + cache_creation_input_token_cost?: number | null + input_cost_per_token_above_200k_tokens?: number | null + output_cost_per_token_above_200k_tokens?: number | null + max_tokens?: number | null + max_input_tokens?: number | null + max_output_tokens?: number | null + supports_function_calling?: boolean | null + supports_vision?: boolean | null + supports_pdf_input?: boolean | null + supports_audio_input?: boolean | null + supports_audio_output?: boolean | null + supports_video_input?: boolean | null + supports_prompt_caching?: boolean | null + supports_reasoning?: boolean | null + supported_openai_params?: string[] | null + [key: string]: unknown + } + } + + const INTERLEAVED_MODELS = ["claude", "anthropic"] + + function isWildcard(name: string): boolean { + return name.includes("*") || name.includes("/*") + } + + function inferInterleaved( + underlyingModel: string | undefined, + ): Provider.Model["capabilities"]["interleaved"] { + if (!underlyingModel) return false + const lower = underlyingModel.toLowerCase() + if (INTERLEAVED_MODELS.some((m) => lower.includes(m))) return true + return false + } + + function costPerMillion(costPerToken: number | null | undefined): number { + if (!costPerToken) return 0 + return costPerToken * 1_000_000 + } + + function toModel(entry: ModelInfoEntry): Provider.Model | undefined { + if (isWildcard(entry.model_name)) return undefined + + const info = entry.model_info ?? {} + const underlyingModel = entry.litellm_params?.model + + const inputCost = costPerMillion(info.input_cost_per_token) + const outputCost = costPerMillion(info.output_cost_per_token) + const cacheReadCost = costPerMillion(info.cache_read_input_token_cost) + const cacheWriteCost = costPerMillion(info.cache_creation_input_token_cost) + + const hasOver200K = + info.input_cost_per_token_above_200k_tokens != null || + info.output_cost_per_token_above_200k_tokens != null + + const supportsVision = info.supports_vision === true + const supportsPdf = info.supports_pdf_input === true + const supportsTemperature = info.supported_openai_params?.includes("temperature") ?? true + + return { + id: entry.model_name, + providerID: "litellm", + name: entry.model_name, + api: { + id: entry.model_name, + url: "", + npm: "@ai-sdk/openai-compatible", + }, + status: "active", + headers: {}, + options: underlyingModel ? { underlyingModel } : {}, + cost: { + input: inputCost, + output: outputCost, + cache: { + read: cacheReadCost, + write: cacheWriteCost, + }, + experimentalOver200K: hasOver200K + ? { + input: costPerMillion(info.input_cost_per_token_above_200k_tokens), + output: costPerMillion(info.output_cost_per_token_above_200k_tokens), + cache: { read: 0, write: 0 }, + } + : undefined, + }, + limit: { + context: (info.max_input_tokens ?? info.max_tokens ?? 128_000) as number, + output: (info.max_output_tokens ?? 8_192) as number, + }, + capabilities: { + temperature: supportsTemperature, + reasoning: info.supports_reasoning === true, + attachment: supportsVision || supportsPdf, + toolcall: info.supports_function_calling !== false, + input: { + text: true, + audio: info.supports_audio_input === true, + image: supportsVision, + video: info.supports_video_input === true, + pdf: supportsPdf, + }, + output: { + text: true, + audio: info.supports_audio_output === true, + image: false, + video: false, + pdf: false, + }, + interleaved: inferInterleaved(underlyingModel), + }, + release_date: "", + variants: {}, + } + } + + function toBasicModel(id: string): Provider.Model { + return { + id, + providerID: "litellm", + name: id, + api: { id, url: "", npm: "@ai-sdk/openai-compatible" }, + status: "active", + headers: {}, + options: {}, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 128_000, output: 8_192 }, + capabilities: { + temperature: true, + reasoning: false, + attachment: false, + toolcall: true, + input: { text: true, audio: false, image: false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + release_date: "", + variants: {}, + } + } + + async function fetchModelInfo( + host: string, + headers: Record, + timeout: number, + ): Promise | undefined> { + const url = `${host}/model/info` + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(timeout), + }).catch(() => undefined) + + if (!response?.ok) return undefined + + const data = (await response.json()) as { data?: ModelInfoEntry[] } + const entries = data?.data + if (!Array.isArray(entries)) return undefined + + const models: Record = {} + for (const entry of entries) { + const model = toModel(entry) + if (model) models[model.id] = model + } + return Object.keys(models).length > 0 ? models : undefined + } + + async function fetchModelList( + host: string, + headers: Record, + timeout: number, + ): Promise> { + const url = `${host}/models` + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(timeout), + }).catch(() => undefined) + + if (!response?.ok) return {} + + const data = (await response.json()) as { data?: { id: string }[] } + const models: Record = {} + for (const item of data?.data ?? []) { + if (!item.id) continue + models[item.id] = toBasicModel(item.id) + } + return models + } + + export async function discover( + host: string, + options?: { + apiKey?: string + headers?: Record + timeout?: number + }, + ): Promise | undefined> { + const timeout = options?.timeout ?? Number(Env.get("LITELLM_TIMEOUT") ?? "5000") + const base = host.replace(/\/+$/, "") + + const headers: Record = { + "Content-Type": "application/json", + ...options?.headers, + } + if (options?.apiKey) { + headers["Authorization"] = `Bearer ${options.apiKey}` + } + + // Try /model/info first for rich metadata, fall back to /models + const rich = await fetchModelInfo(base, headers, timeout) + if (rich) { + log.info("discovered models from LiteLLM /model/info", { + count: Object.keys(rich).length, + host, + }) + return rich + } + + const basic = await fetchModelList(base, headers, timeout) + if (Object.keys(basic).length > 0) { + log.info("discovered models from /models (fallback)", { + count: Object.keys(basic).length, + host, + }) + return basic + } + + return undefined + } +} diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 022ec316795..fb1b919ee33 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -44,6 +44,7 @@ import { fromNodeProviderChain } from "@aws-sdk/credential-providers" import { GoogleAuth } from "google-auth-library" import { ProviderTransform } from "./transform" import { Installation } from "../installation" +import { LiteLLM } from "./litellm" export namespace Provider { const log = Log.create({ service: "provider" }) @@ -588,6 +589,72 @@ export namespace Provider { }, } }, + litellm: async (provider) => { + const config = await Config.get() + const providerConfig = config.provider?.["litellm"] + + const auth = await Auth.get("litellm") + + // Skip discovery when there is no configuration at all + if ( + !providerConfig && + !Env.get("LITELLM_API_KEY") && + !Env.get("LITELLM_HOST") && + !Env.get("LITELLM_BASE_URL") && + auth?.type !== "api" + ) + return { autoload: false } + + const baseURL = + providerConfig?.options?.baseURL ?? + Env.get("LITELLM_HOST") ?? + Env.get("LITELLM_BASE_URL") ?? + "http://localhost:4000" + + const apiKey = await (async () => { + if (providerConfig?.options?.apiKey) return providerConfig.options.apiKey + const envKey = Env.get("LITELLM_API_KEY") + if (envKey) return envKey + if (auth?.type === "api") return auth.key + return undefined + })() + + const customHeaders = await Promise.resolve(Env.get("LITELLM_CUSTOM_HEADERS")) + .then((raw) => (raw ? (JSON.parse(raw) as Record) : {})) + .catch(() => ({})) + + const timeout = Number(Env.get("LITELLM_TIMEOUT") ?? "5000") + + const discovered = await LiteLLM.discover(baseURL, { + apiKey, + headers: customHeaders, + timeout, + }) + + if (discovered) { + for (const [modelID, model] of Object.entries(discovered)) { + if (!provider.models[modelID]) { + provider.models[modelID] = model + } + } + } + + const hasModels = Object.keys(provider.models).length > 0 + if (!hasModels) return { autoload: false } + + return { + autoload: true, + options: { + baseURL, + apiKey, + litellmProxy: true, + ...customHeaders, + }, + async getModel(sdk: any, modelID: string) { + return sdk.languageModel(modelID) + }, + } + }, } export const Model = z @@ -794,6 +861,18 @@ export namespace Provider { } } + // Seed LiteLLM provider so it is always available for interactive configuration + if (!database["litellm"]) { + database["litellm"] = { + id: "litellm", + name: "LiteLLM", + env: ["LITELLM_API_KEY"], + options: {}, + source: "custom", + models: {}, + } + } + function mergeProvider(providerID: string, provider: Partial) { const existing = providers[providerID] if (existing) { diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts index b659799c1b6..aa34e8e3bd1 100644 --- a/packages/opencode/src/provider/transform.ts +++ b/packages/opencode/src/provider/transform.ts @@ -71,7 +71,11 @@ export namespace ProviderTransform { .filter((msg): msg is ModelMessage => msg !== undefined && msg.content !== "") } - if (model.api.id.includes("claude")) { + const isClaudeToolCall = + model.api.id.includes("claude") || + (model.providerID === "litellm" && + ((model.options?.underlyingModel as string) ?? "").toLowerCase().includes("claude")) + if (isClaudeToolCall) { return msgs.map((msg) => { if ((msg.role === "assistant" || msg.role === "tool") && Array.isArray(msg.content)) { msg.content = msg.content.map((part) => { @@ -252,13 +256,17 @@ export namespace ProviderTransform { export function message(msgs: ModelMessage[], model: Provider.Model, options: Record) { msgs = unsupportedParts(msgs, model) msgs = normalizeMessages(msgs, model, options) + const underlying = ((model.options?.underlyingModel as string) ?? "").toLowerCase() + const isLiteLLMClaude = + model.providerID === "litellm" && (underlying.includes("claude") || underlying.includes("anthropic")) if ( (model.providerID === "anthropic" || model.api.id.includes("anthropic") || model.api.id.includes("claude") || model.id.includes("anthropic") || model.id.includes("claude") || - model.api.npm === "@ai-sdk/anthropic") && + model.api.npm === "@ai-sdk/anthropic" || + isLiteLLMClaude) && model.api.npm !== "@ai-sdk/gateway" ) { msgs = applyCaching(msgs, model) @@ -363,6 +371,48 @@ export namespace ProviderTransform { } if (id.includes("grok")) return {} + // LiteLLM proxied models: infer reasoning variant from the underlying model + // to avoid false-positive reasoning param injection for aliased models. + // Model aliases (e.g., "sonnet-4.5") may not contain "claude" or "anthropic", + // so we also check the underlying model stored in options.underlyingModel + // (e.g., "azure_ai/claude-sonnet-4-5"). + // NOTE: @ai-sdk/openai-compatible passes provider options as raw request body + // fields, so we must use snake_case (budget_tokens) not camelCase (budgetTokens) + // since LiteLLM forwards them directly to the upstream API. + if (model.providerID === "litellm" && model.capabilities.reasoning) { + const apiId = model.api.id.toLowerCase() + const underlying = ((model.options?.underlyingModel as string) ?? "").toLowerCase() + const isClaude = [apiId, underlying].some((s) => s.includes("claude") || s.includes("anthropic")) + if (isClaude) { + const isAdaptive = ["opus-4-6", "opus-4.6", "sonnet-4-6", "sonnet-4.6"].some( + (v) => apiId.includes(v) || underlying.includes(v), + ) + if (isAdaptive) { + return Object.fromEntries( + ["low", "medium", "high", "max"].map((effort) => [ + effort, + { thinking: { type: "enabled", budget_tokens: Math.min(16_000, Math.floor(model.limit.output / 2 - 1)) } }, + ]), + ) + } + return { + high: { + thinking: { + type: "enabled", + budget_tokens: Math.min(16_000, Math.floor(model.limit.output / 2 - 1)), + }, + }, + max: { + thinking: { + type: "enabled", + budget_tokens: Math.min(31_999, model.limit.output - 1), + }, + }, + } + } + return Object.fromEntries(WIDELY_SUPPORTED_EFFORTS.map((effort) => [effort, { reasoningEffort: effort }])) + } + switch (model.api.npm) { case "@openrouter/ai-sdk-provider": if (!model.id.includes("gpt") && !model.id.includes("gemini-3") && !model.id.includes("claude")) return {} diff --git a/packages/opencode/src/server/routes/provider.ts b/packages/opencode/src/server/routes/provider.ts index 872b48be79d..3f53f8d00ae 100644 --- a/packages/opencode/src/server/routes/provider.ts +++ b/packages/opencode/src/server/routes/provider.ts @@ -40,6 +40,17 @@ export const ProviderRoutes = lazy(() => const enabled = config.enabled_providers ? new Set(config.enabled_providers) : undefined const allProviders = await ModelsDev.get() + + // Include LiteLLM so it is available for interactive configuration + if (!allProviders["litellm"]) { + allProviders["litellm"] = { + id: "litellm", + name: "LiteLLM", + env: ["LITELLM_API_KEY"], + models: {}, + } as (typeof allProviders)["string"] + } + const filteredProviders: Record = {} for (const [key, value] of Object.entries(allProviders)) { if ((enabled ? enabled.has(key) : true) && !disabled.has(key)) { @@ -54,7 +65,7 @@ export const ProviderRoutes = lazy(() => ) return c.json({ all: Object.values(providers), - default: mapValues(providers, (item) => Provider.sort(Object.values(item.models))[0].id), + default: mapValues(providers, (item) => Provider.sort(Object.values(item.models))[0]?.id), connected: Object.keys(connected), }) }, diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 4e42fb0d2ec..c0810df4fda 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -156,6 +156,7 @@ export namespace LLM { // 1. Providers with "litellm" in their ID or API ID (auto-detected) // 2. Providers with explicit "litellmProxy: true" option (opt-in for custom gateways) const isLiteLLMProxy = + input.model.providerID === "litellm" || provider.options?.["litellmProxy"] === true || input.model.providerID.toLowerCase().includes("litellm") || input.model.api.id.toLowerCase().includes("litellm") @@ -199,7 +200,7 @@ export namespace LLM { temperature: params.temperature, topP: params.topP, topK: params.topK, - providerOptions: ProviderTransform.providerOptions(input.model, params.options), + providerOptions: ProviderTransform.providerOptions(input.model, filterInternalOptions(params.options)), activeTools: Object.keys(tools).filter((x) => x !== "invalid"), tools, toolChoice: input.toolChoice, @@ -265,6 +266,14 @@ export namespace LLM { return input.tools } + // Filter internal metadata keys from options before passing to providerOptions. + // These keys are used for internal logic (e.g., variant detection) but should not + // be sent as request body fields to the provider API. + function filterInternalOptions(options: Record): Record { + const { underlyingModel, ...rest } = options + return rest + } + // Check if messages contain any tool-call content // Used to determine if a dummy tool should be added for LiteLLM proxy compatibility export function hasToolCalls(messages: ModelMessage[]): boolean { diff --git a/packages/opencode/src/session/system.ts b/packages/opencode/src/session/system.ts index a61dd8cba55..b7b999f9aee 100644 --- a/packages/opencode/src/session/system.ts +++ b/packages/opencode/src/session/system.ts @@ -17,12 +17,16 @@ export namespace SystemPrompt { } export function provider(model: Provider.Model) { - if (model.api.id.includes("gpt-5")) return [PROMPT_CODEX] - if (model.api.id.includes("gpt-") || model.api.id.includes("o1") || model.api.id.includes("o3")) - return [PROMPT_BEAST] - if (model.api.id.includes("gemini-")) return [PROMPT_GEMINI] - if (model.api.id.includes("claude")) return [PROMPT_ANTHROPIC] - if (model.api.id.toLowerCase().includes("trinity")) return [PROMPT_TRINITY] + const apiId = model.api.id + const underlying = ((model.options?.underlyingModel as string) ?? "").toLowerCase() + const isLiteLLMClaude = + model.providerID === "litellm" && (underlying.includes("claude") || underlying.includes("anthropic")) + + if (apiId.includes("gpt-5")) return [PROMPT_CODEX] + if (apiId.includes("gpt-") || apiId.includes("o1") || apiId.includes("o3")) return [PROMPT_BEAST] + if (apiId.includes("gemini-")) return [PROMPT_GEMINI] + if (apiId.includes("claude") || isLiteLLMClaude) return [PROMPT_ANTHROPIC] + if (apiId.toLowerCase().includes("trinity")) return [PROMPT_TRINITY] return [PROMPT_ANTHROPIC_WITHOUT_TODO] }