/** * Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq, * Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the * Ollama adapter. * * These servers agree on the basics and disagree on everything else, so this * adapter is deliberately forgiving: * - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs * `max_completion_tokens` and `reasoning_effort` are each dropped and retried * once when a server answers 400 about them; * - usage may be missing, and may arrive on a chunk with no choices; * - tool-call argument fragments may arrive in any chunking, with or without ids; * - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`; * - the last chunk may lack a `finish_reason`. */ import { type EmbedRequest, type EmbedResult, AIError, type ChatMessage, type ChatRequest, type ContentPart, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StopReason, type StreamEvent, type TestResult, } from '../types.ts'; import { parseEmbeddings, dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts'; import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts'; type Json = Record; /** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */ export function normalizeCompatBase(url: string): string { let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, ''); try { const u = new URL(base); // A bare host ("http://localhost:1234") almost always means the conventional /v1 root. if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`; } catch { /* validated by the caller */ } return base; } export interface CompatOptions { kind: ProviderKind; label: string; baseUrl: string; apiKey: string | null; headers?: Record; } function userContent(parts: ContentPart[], caps: Partial): string | Json[] { const blocks: Json[] = []; for (const p of parts) { if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text }); else if (p.type === 'image') { if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' }); else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } }); } else if (p.type === 'file') { if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') }); // Chat Completions "file" parts (OpenAI, OpenRouter and others) for models marked as reading PDFs. else if (p.mime === 'application/pdf' && caps.pdf) blocks.push({ type: 'file', file: { filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) } }); else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` }); } } // Plain strings are understood by every server; arrays only by multimodal ones. if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n'); return blocks; } export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] { const caps = req.capabilities ?? {}; const out: Json[] = []; if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() }); for (const msg of req.messages) { if (msg.role === 'assistant') { const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join(''); const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract[]; if (!text && calls.length === 0) continue; const wire: Json = { role: 'assistant', content: text || null }; if (calls.length > 0) { wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } })); } out.push(wire); continue; } // Tool results must directly follow the assistant turn that requested them. for (const p of msg.content) { if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' }); } const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call'); if (rest.length > 0) { const content = userContent(rest, caps); if (typeof content !== 'string' || content) out.push({ role: 'user', content }); } } return out; } interface BodyFlags { tools: boolean; usage: boolean; jsonMode: boolean; temperature: boolean; reasoning: boolean; maxKey: 'max_tokens' | 'max_completion_tokens' | null; } function buildBody(req: ChatRequest, flags: BodyFlags): Json { const caps = req.capabilities ?? {}; const body: Json = { model: req.model, messages: toChatMessages(req), stream: true }; if (flags.usage) body.stream_options = { include_usage: true }; if (flags.tools && caps.tools !== false && req.tools?.length) { body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } })); const choice = req.toolChoice; if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice; else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } }; } if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature; if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens; if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort; if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' }; return body; } function mapFinish(reason: string): StopReason { switch (reason) { case 'stop': case 'eos': case 'end_turn': return 'end'; case 'tool_calls': case 'function_call': return 'tool_use'; case 'length': case 'max_tokens': return 'max_tokens'; case 'content_filter': return 'refusal'; default: return 'other'; } } function authHeaders(o: CompatOptions): Record { const h: Record = { ...o.headers }; if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`; return h; } function errorContext(o: CompatOptions): ErrorContext { return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] }; } export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator { const ctx = errorContext(o); const base = normalizeCompatBase(o.baseUrl); const local = isLocalUrl(base); const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' }; let http: HttpResponse; for (let attempt = 0; ; attempt += 1) { http = await httpRequest(`${base}/chat/completions`, { ...ctx, headers: authHeaders(o), body: buildBody(req, flags), signal: req.signal, // A local server may spend minutes loading a model into memory before the first byte. firstByteTimeoutMs: local ? 10 * 60_000 : undefined, }); try { await ensureOk(http); break; } catch (err) { if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err; const detail = (err.providerMessage ?? '').toLowerCase(); if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false; else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens'; else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null; else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false; else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false; else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false; else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) { // The model (or server) cannot do tool calling at all. if (req.toolChoice === 'required' || isRecord(req.toolChoice)) { throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, { status: err.status, providerMessage: err.providerMessage, }); } flags.tools = false; } else throw err; } } let text = ''; let finish = ''; let reasoningSeen = false; const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 }; const calls: { id: string; name: string; args: string }[] = []; for await (const { data } of sseJson(http)) { if (!isRecord(data)) continue; if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data); if (isRecord(data.usage)) { const u = data.usage; usage.input = num(u.prompt_tokens) || usage.input; usage.output = num(u.completion_tokens) || usage.output; if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens); if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens); } const choice = Array.isArray(data.choices) ? data.choices[0] : undefined; if (!isRecord(choice)) continue; if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason; const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null; if (!delta) continue; const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking); if (reasoning) { reasoningSeen = true; yield { type: 'reasoning', delta: reasoning }; } const piece = typeof delta.content === 'string' ? delta.content : ''; if (piece) { text += piece; yield { type: 'text', delta: piece }; } if (Array.isArray(delta.tool_calls)) { for (const raw of delta.tool_calls) { if (!isRecord(raw)) continue; const fn = isRecord(raw.function) ? raw.function : {}; // `index` identifies the call; servers that omit it send one whole call per entry. const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1; let slot = index >= 0 ? calls[index] : undefined; if (!slot) { slot = { id: '', name: '', args: '' }; if (index >= 0) calls[index] = slot; else calls.push(slot); } if (str(raw.id)) slot.id = str(raw.id); if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name); if (typeof fn.arguments === 'string') slot.args += fn.arguments; else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments); } } } const stop = finish ? mapFinish(finish) : 'end'; const content: ContentPart[] = []; if (text) content.push({ type: 'text', text }); const done: { id: string; name: string; input: Record }[] = []; for (const c of calls) { if (!c || !c.name) continue; let input = parseArgsStrict(c.args); if (!input) { if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest input = parseToolArguments(c.args, o.label, c.name); } done.push({ id: c.id || newToolCallId('call'), name: c.name, input }); } for (const call of done) { content.push({ type: 'tool_call', ...call }); yield { type: 'tool_call', call }; } if (content.length === 0 && !reasoningSeen && !finish) { throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true }); } const message: ChatMessage = { role: 'assistant', content }; yield { type: 'done', message, stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop, usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }), }; } export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise { const base = normalizeCompatBase(o.baseUrl); let res: unknown; try { res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs }); } catch (err) { if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) { throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status }); } throw err; } const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : []; const found: DiscoveredModel[] = []; for (const m of list) { if (!isRecord(m)) continue; const id = str(m.id) || str(m.name); if (!id || !isChatModelId(id)) continue; const model: DiscoveredModel = { id }; if (str(m.name) && str(m.name) !== id) model.label = str(m.name); // OpenRouter-style metadata, when present. const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length); if (ctxLen) model.contextWindow = ctxLen; if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens); if (isRecord(m.pricing)) { const pin = Number(m.pricing.prompt); const pout = Number(m.pricing.completion); if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3; if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3; } const caps: Partial = {}; if (Array.isArray(m.supported_parameters)) { caps.tools = m.supported_parameters.includes('tools'); caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort'); } if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) { caps.vision = m.architecture.input_modalities.includes('image'); } if (Object.keys(caps).length > 0) model.capabilities = caps; found.push(model); } return withCatalogDefaults(o.kind, found); } /** OpenAI-style `/embeddings`, shared with Ollama's compatible endpoint. */ export async function compatEmbed(o: CompatOptions, req: EmbedRequest): Promise { const body: Record = { model: req.model, input: req.inputs, encoding_format: 'float' }; if (req.dimensions && /text-embedding-3/.test(req.model)) body.dimensions = req.dimensions; const res = await requestJson(`${normalizeCompatBase(o.baseUrl)}/embeddings`, { ...errorContext(o), headers: authHeaders(o), body, signal: req.signal, firstByteTimeoutMs: 120_000, totalTimeoutMs: 300_000 }); return parseEmbeddings(res, req.inputs.length, o.label); } function options(cfg: ProviderConfig): CompatOptions { if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`); return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers }; } async function test(cfg: ProviderConfig, model?: string): Promise { return runTest(cfg.name || 'the server', async () => { const o = options(cfg); try { return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length }; } catch (err) { if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err; // No model listing: prove the connection with a one-token completion instead. const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS); try { for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) { if (ev.type === 'done') break; } } finally { clearTimeout(timer); } return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` }; } }); } export const openaiCompatibleAdapter: ProviderAdapter = { kind: 'openai_compatible', meta: { label: 'OpenAI-compatible', defaultBaseUrl: null, requiresApiKey: false, requiresBaseUrl: true, supportsDiscovery: true, docsUrl: 'https://openrouter.ai/docs/quickstart', keyHint: 'Optional for local servers', }, stream: (cfg, req) => streamChatCompletions(options(cfg), req), listModels: (cfg, signal) => listCompatModels(options(cfg), signal), test, embed: (cfg, req) => compatEmbed(options(cfg), req), };