diff --git a/packages/opencode/src/provider/litellm.ts b/packages/opencode/src/provider/litellm.ts new file mode 100644 index 00000000000..bf5431aeed6 --- /dev/null +++ b/packages/opencode/src/provider/litellm.ts @@ -0,0 +1,170 @@ +import { Log } from "../util/log" +import { Env } from "../env" +import type { Provider } from "./provider" + +export namespace LiteLLM { + const log = Log.create({ service: "litellm" }) + + interface ModelInfoEntry { + model_name: string + litellm_params?: { + model?: string + [key: string]: unknown + } + model_info?: { + id?: string + input_cost_per_token?: number + output_cost_per_token?: number + cache_read_input_token_cost?: number + cache_creation_input_token_cost?: number + max_tokens?: number + max_input_tokens?: number + max_output_tokens?: number + supports_function_calling?: boolean + supports_vision?: boolean + supports_pdf_input?: boolean + supports_audio_input?: boolean + supports_audio_output?: boolean + supports_video_input?: boolean + supports_prompt_caching?: boolean + supports_reasoning?: boolean + [key: string]: unknown + } + } + + const INTERLEAVED_MODELS = ["claude", "anthropic"] + + function isWildcard(name: string): boolean { + return name.includes("*") || name.includes("/*") + } + + function inferInterleaved( + underlyingModel: string | undefined, + ): Provider.Model["capabilities"]["interleaved"] { + if (!underlyingModel) return false + const lower = underlyingModel.toLowerCase() + if (INTERLEAVED_MODELS.some((m) => lower.includes(m))) return true + return false + } + + function toModel(entry: ModelInfoEntry): Provider.Model | undefined { + if (isWildcard(entry.model_name)) return undefined + + const info = entry.model_info ?? {} + const underlyingModel = entry.litellm_params?.model + + const inputCost = (info.input_cost_per_token ?? 0) * 1_000_000 + const outputCost = (info.output_cost_per_token ?? 0) * 1_000_000 + const cacheReadCost = (info.cache_read_input_token_cost ?? 0) * 1_000_000 + const cacheWriteCost = (info.cache_creation_input_token_cost ?? 0) * 1_000_000 + + return { + id: entry.model_name, + providerID: "litellm", + name: entry.model_name, + api: { + id: entry.model_name, + url: "", + npm: "@ai-sdk/openai-compatible", + }, + status: "active", + headers: {}, + options: {}, + cost: { + input: inputCost, + output: outputCost, + cache: { + read: cacheReadCost, + write: cacheWriteCost, + }, + }, + limit: { + context: (info.max_tokens ?? info.max_input_tokens ?? 128_000) as number, + output: (info.max_output_tokens ?? 8_192) as number, + }, + capabilities: { + temperature: true, + reasoning: info.supports_reasoning ?? false, + attachment: (info.supports_vision || info.supports_pdf_input) ?? false, + toolcall: info.supports_function_calling ?? true, + input: { + text: true, + audio: info.supports_audio_input ?? false, + image: info.supports_vision ?? false, + video: info.supports_video_input ?? false, + pdf: info.supports_pdf_input ?? false, + }, + output: { + text: true, + audio: info.supports_audio_output ?? false, + image: false, + video: false, + pdf: false, + }, + interleaved: inferInterleaved(underlyingModel), + }, + release_date: "", + variants: {}, + } + } + + export async function discover( + host: string, + options?: { + apiKey?: string + headers?: Record + timeout?: number + }, + ): Promise | undefined> { + const timeout = options?.timeout ?? Number(Env.get("LITELLM_TIMEOUT") ?? "5000") + const url = `${host.replace(/\/+$/, "")}/model/info` + + try { + const headers: Record = { + "Content-Type": "application/json", + ...options?.headers, + } + if (options?.apiKey) { + headers["Authorization"] = `Bearer ${options.apiKey}` + } + + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(timeout), + }) + + if (!response.ok) { + log.warn("LiteLLM model discovery failed", { + status: response.status, + url, + }) + return undefined + } + + const data = (await response.json()) as { data?: ModelInfoEntry[] } + const entries = data?.data + if (!Array.isArray(entries)) { + log.warn("LiteLLM /model/info returned unexpected format", { url }) + return undefined + } + + const models: Record = {} + for (const entry of entries) { + const model = toModel(entry) + if (model) { + models[model.id] = model + } + } + + log.info("discovered models from LiteLLM proxy", { + count: Object.keys(models).length, + host, + }) + + return models + } catch (e) { + log.warn("LiteLLM model discovery error", { error: e, url }) + return undefined + } + } +} diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index d94d0cbb223..09a328c35c8 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -41,6 +41,7 @@ import { createVercel } from "@ai-sdk/vercel" import { createGitLab, VERSION as GITLAB_PROVIDER_VERSION } from "@gitlab/gitlab-ai-provider" import { ProviderTransform } from "./transform" import { Installation } from "../installation" +import { LiteLLM } from "./litellm" export namespace Provider { const log = Log.create({ service: "provider" }) @@ -589,6 +590,68 @@ export namespace Provider { }, } }, + litellm: async (provider) => { + const config = await Config.get() + const providerConfig = config.provider?.["litellm"] + + const baseURL = + providerConfig?.options?.baseURL ?? + Env.get("LITELLM_HOST") ?? + Env.get("LITELLM_BASE_URL") ?? + "http://localhost:4000" + + const apiKey = await (async () => { + if (providerConfig?.options?.apiKey) return providerConfig.options.apiKey + const envKey = Env.get("LITELLM_API_KEY") + if (envKey) return envKey + const auth = await Auth.get("litellm") + if (auth?.type === "api") return auth.key + return undefined + })() + + const customHeaders = iife(() => { + const raw = Env.get("LITELLM_CUSTOM_HEADERS") + if (!raw) return {} + try { + return JSON.parse(raw) as Record + } catch { + return {} + } + }) + + const timeout = Number(Env.get("LITELLM_TIMEOUT") ?? "5000") + + const discovered = await LiteLLM.discover(baseURL, { + apiKey, + headers: customHeaders, + timeout, + }) + + if (discovered) { + // Inject discovered models; user config models take precedence + for (const [modelID, model] of Object.entries(discovered)) { + if (!provider.models[modelID]) { + provider.models[modelID] = model + } + } + } + + const hasModels = Object.keys(provider.models).length > 0 + if (!hasModels) return { autoload: false } + + return { + autoload: true, + options: { + baseURL, + apiKey, + litellmProxy: true, + ...customHeaders, + }, + async getModel(sdk: any, modelID: string) { + return sdk.languageModel(modelID) + }, + } + }, } export const Model = z @@ -795,6 +858,18 @@ export namespace Provider { } } + // Seed LiteLLM provider when env vars exist but no entry in database + if (!database["litellm"] && (Env.get("LITELLM_API_KEY") || Env.get("LITELLM_HOST") || Env.get("LITELLM_BASE_URL"))) { + database["litellm"] = { + id: "litellm", + name: "LiteLLM", + env: ["LITELLM_API_KEY"], + options: {}, + source: "custom", + models: {}, + } + } + function mergeProvider(providerID: string, provider: Partial) { const existing = providers[providerID] if (existing) { diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts index 759dab440d4..5749dcccbc1 100644 --- a/packages/opencode/src/provider/transform.ts +++ b/packages/opencode/src/provider/transform.ts @@ -359,6 +359,29 @@ export namespace ProviderTransform { } if (id.includes("grok")) return {} + // LiteLLM proxied models: infer reasoning variant from the underlying model + // to avoid false-positive reasoning param injection for aliased models + if (model.providerID === "litellm" && model.capabilities.reasoning) { + const apiId = model.api.id.toLowerCase() + if (apiId.includes("claude") || apiId.includes("anthropic")) { + return { + high: { + thinking: { + type: "enabled", + budgetTokens: Math.min(16_000, Math.floor(model.limit.output / 2 - 1)), + }, + }, + max: { + thinking: { + type: "enabled", + budgetTokens: Math.min(31_999, model.limit.output - 1), + }, + }, + } + } + return Object.fromEntries(WIDELY_SUPPORTED_EFFORTS.map((effort) => [effort, { reasoningEffort: effort }])) + } + switch (model.api.npm) { case "@openrouter/ai-sdk-provider": if (!model.id.includes("gpt") && !model.id.includes("gemini-3") && !model.id.includes("claude")) return {} diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index fa880391276..5b95684ed94 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -160,6 +160,7 @@ export namespace LLM { // 1. Providers with "litellm" in their ID or API ID (auto-detected) // 2. Providers with explicit "litellmProxy: true" option (opt-in for custom gateways) const isLiteLLMProxy = + input.model.providerID === "litellm" || provider.options?.["litellmProxy"] === true || input.model.providerID.toLowerCase().includes("litellm") || input.model.api.id.toLowerCase().includes("litellm")