Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
581
server/ai/providers/anthropic.ts
Normal file
581
server/ai/providers/anthropic.ts
Normal file
|
|
@ -0,0 +1,581 @@
|
|||
/**
|
||||
* Anthropic — Messages API (`POST /v1/messages`, SSE).
|
||||
*
|
||||
* Verified against (2026-11):
|
||||
* https://platform.claude.com/docs/en/api/messages
|
||||
* https://platform.claude.com/docs/en/build-with-claude/streaming
|
||||
* https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
|
||||
* https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
|
||||
* https://platform.claude.com/docs/en/api/migration-2026-10-21
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - API version 2026-10-21 rejects temperature/top_p/top_k on EVERY model, so
|
||||
* this adapter never sends a temperature (the catalog marks Claude models
|
||||
* `temperature: false`).
|
||||
* - `max_tokens` is required; thinking counts against it.
|
||||
* - Thinking configuration differs per model generation (see `claudeProfile`).
|
||||
* - Thinking blocks (with their signatures) must be replayed unchanged, so the
|
||||
* raw content blocks of each turn are kept in `providerData`.
|
||||
* - Newer models bind thinking blocks to the conversation prefix (system
|
||||
* prompt, tool set, earlier messages). If the app changed any of those, the
|
||||
* API answers 400 "Invalid `signature` in `thinking` block"; the documented
|
||||
* recovery is to strip the thinking blocks and retry once.
|
||||
* - Safety classifiers can end a turn with `stop_reason: "refusal"` (HTTP 200).
|
||||
* For the models that support it we opt into server-side fallback, which
|
||||
* re-runs a declined request on another Claude model inside the same call.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'Anthropic';
|
||||
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
|
||||
const API_VERSION = '2026-10-21';
|
||||
/** Beta that enables the `fallbacks` request parameter. */
|
||||
const FALLBACK_BETA = 'server-side-fallback-2026-07-01';
|
||||
/** Streaming has no HTTP-timeout concern, so give the model room (thinking counts against this). */
|
||||
const DEFAULT_MAX_TOKENS = 64_000;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-model behaviour */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface ClaudeProfile {
|
||||
/**
|
||||
* always – thinking cannot be turned off (Fable/Mythos, Opus 5.5+)
|
||||
* default_on – adaptive thinking runs unless disabled (Opus 5, Sonnet 5+, Haiku 5.5)
|
||||
* opt_in – adaptive thinking must be requested (Opus 4.6–4.8, Sonnet 4.6)
|
||||
* budget – legacy `budget_tokens` thinking (Haiku 4.5, Opus/Sonnet 4.5 and older)
|
||||
* none – no thinking support (Claude 3 and earlier)
|
||||
*/
|
||||
thinking: 'always' | 'default_on' | 'opt_in' | 'budget' | 'none';
|
||||
/** How thinking is switched off, when the model allows it. */
|
||||
off: 'disabled' | 'between_tools' | null;
|
||||
/** Accepts `output_config.effort`. */
|
||||
effort: boolean;
|
||||
/** Accepts `thinking.display` (and defaults to hiding the reasoning text). */
|
||||
display: boolean;
|
||||
/** Accepts `tool_choice` of type `any` / `tool`. */
|
||||
forcedToolChoice: boolean;
|
||||
/** Supports the server-side refusal `fallbacks` parameter. */
|
||||
fallbacks: boolean;
|
||||
}
|
||||
|
||||
/** Thinking / tool-choice rules by model, from the model migration guides. */
|
||||
export function claudeProfile(model: string): ClaudeProfile {
|
||||
const m = /claude-(fable|mythos|opus|sonnet|haiku)-(\d+)(?:-(\d{1,2}))?(?!\d)/.exec(model);
|
||||
if (!m) {
|
||||
// "claude-3-5-sonnet-…" and older put the version first and never had adaptive thinking.
|
||||
if (/claude-(?:[123]\b|instant)/.test(model)) {
|
||||
return { thinking: 'none', off: null, effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
// Unknown naming scheme: assume it behaves like the newest generation.
|
||||
return { thinking: 'default_on', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: false };
|
||||
}
|
||||
const family = m[1];
|
||||
const v = Number(m[2]) * 10 + Number(m[3] ?? 0); // 4.6 -> 46, 5 -> 50, 5.5 -> 55
|
||||
|
||||
if (family === 'fable' || family === 'mythos') {
|
||||
return {
|
||||
thinking: 'always',
|
||||
off: null,
|
||||
effort: true,
|
||||
display: true,
|
||||
forcedToolChoice: v <= 50,
|
||||
fallbacks: family === 'fable' && (v === 51 || v === 52),
|
||||
};
|
||||
}
|
||||
if (family === 'opus') {
|
||||
if (v >= 55) return { thinking: 'always', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: v === 50 };
|
||||
if (v >= 47) return { thinking: 'opt_in', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: v === 45, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
if (family === 'sonnet') {
|
||||
// Sonnet 5.5 rejects {type:"disabled"}; its lowest setting is {type:"between_tools"}.
|
||||
if (v >= 55) return { thinking: 'default_on', off: 'between_tools', effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
// haiku
|
||||
if (v >= 55) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
|
||||
/** Thinking budgets for the legacy `budget_tokens` models. */
|
||||
const LEGACY_BUDGET = { low: 2_048, medium: 8_192, high: 16_384 } as const;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Request building */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
interface BuildOptions {
|
||||
/** Official endpoint: enables features a proxy may not understand. */
|
||||
official: boolean;
|
||||
stripThinking: boolean;
|
||||
useFallbacks: boolean;
|
||||
eagerToolInput: boolean;
|
||||
promptCache: boolean;
|
||||
}
|
||||
|
||||
/** Tool-call ids must match ^[a-zA-Z0-9_-]+$; ids minted by other providers may not. */
|
||||
function safeId(id: string): string {
|
||||
const cleaned = id.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return cleaned || 'tool_call';
|
||||
}
|
||||
|
||||
function userBlocks(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const results: Json[] = [];
|
||||
const rest: Json[] = [];
|
||||
for (const p of parts) {
|
||||
switch (p.type) {
|
||||
case 'tool_result': {
|
||||
// All results of one assistant turn go into ONE user message, before any other content.
|
||||
const block: Json = { type: 'tool_result', tool_use_id: safeId(p.toolCallId), content: p.output || '(no output)' };
|
||||
if (p.isError) block.is_error = true;
|
||||
results.push(block);
|
||||
break;
|
||||
}
|
||||
case 'text':
|
||||
if (p.text) rest.push({ type: 'text', text: p.text });
|
||||
break;
|
||||
case 'image':
|
||||
if (caps.vision === false) rest.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else rest.push({ type: 'image', source: { type: 'base64', media_type: p.mime, data: p.data } });
|
||||
break;
|
||||
case 'file':
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) {
|
||||
const doc: Json = { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: p.data } };
|
||||
if (p.filename) doc.title = p.filename;
|
||||
rest.push(doc);
|
||||
} else if (p.mime.startsWith('text/')) {
|
||||
const doc: Json = { type: 'document', source: { type: 'text', media_type: 'text/plain', data: Buffer.from(p.data, 'base64').toString('utf8') } };
|
||||
if (p.filename) doc.title = p.filename;
|
||||
rest.push(doc);
|
||||
} else {
|
||||
rest.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
break;
|
||||
case 'tool_call':
|
||||
break; // not valid in a user message
|
||||
}
|
||||
}
|
||||
return [...results, ...rest];
|
||||
}
|
||||
|
||||
function assistantBlocks(msg: ChatMessage, stripThinking: boolean): Json[] {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'anthropic' && isRecord(pd.data) && Array.isArray(pd.data.content)) {
|
||||
// Same provider: replay the turn exactly as it was received, thinking blocks
|
||||
// and signatures included.
|
||||
const blocks = pd.data.content.filter(isRecord);
|
||||
return stripThinking ? blocks.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking') : blocks;
|
||||
}
|
||||
// Another provider produced this turn: rebuild it from the neutral parts.
|
||||
const blocks: Json[] = [];
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
||||
else if (p.type === 'tool_call') blocks.push({ type: 'tool_use', id: safeId(p.id), name: p.name, input: p.input });
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function toWireMessages(messages: ChatMessage[], caps: Partial<ModelCapabilities>, stripThinking: boolean): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
const content = msg.role === 'user' ? userBlocks(msg.content, caps) : assistantBlocks(msg, stripThinking);
|
||||
if (content.length === 0) continue; // the API rejects empty turns
|
||||
out.push({ role: msg.role, content });
|
||||
}
|
||||
if (out[0]?.role === 'assistant') out.unshift({ role: 'user', content: [{ type: 'text', text: '(continuing the conversation)' }] });
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, profile: ClaudeProfile, opts: BuildOptions): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const defaults = lookupModelDefaults('anthropic', req.model);
|
||||
const ceiling = defaults.maxOutput ?? Number.POSITIVE_INFINITY;
|
||||
const maxTokens = Math.max(1, Math.min(req.maxOutputTokens ?? DEFAULT_MAX_TOKENS, ceiling));
|
||||
|
||||
const systemNotes: string[] = [];
|
||||
const body: Json = { model: req.model, max_tokens: maxTokens, stream: true };
|
||||
|
||||
// --- tools -------------------------------------------------------
|
||||
const tools = caps.tools === false ? [] : (req.tools ?? []);
|
||||
let forcing = false;
|
||||
if (tools.length > 0) {
|
||||
body.tools = tools.map((t) => {
|
||||
const tool: Json = { name: t.name, description: t.description, input_schema: { type: 'object', ...t.parameters } };
|
||||
// Without this the API buffers each tool-input parameter and the stream goes
|
||||
// silent for the whole time a large input (a generated quiz, say) is written.
|
||||
// The price is that the input is no longer validated server-side, so it is
|
||||
// parsed strictly below and truncated calls are dropped.
|
||||
if (opts.eagerToolInput) tool.eager_input_streaming = true;
|
||||
return tool;
|
||||
});
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none') {
|
||||
body.tool_choice = { type: 'none' };
|
||||
} else if (choice === 'required' || isRecord(choice)) {
|
||||
const name = isRecord(choice) ? str(choice.name) : '';
|
||||
if (profile.forcedToolChoice) {
|
||||
body.tool_choice = name ? { type: 'tool', name } : { type: 'any' };
|
||||
forcing = true;
|
||||
} else {
|
||||
// Fable 5.1+, Opus 5.5 and Sonnet 5.5 answer 400 to forced tool use; the
|
||||
// documented replacement is `auto` plus an explicit instruction.
|
||||
systemNotes.push(
|
||||
name
|
||||
? `For this turn you must call the \`${name}\` tool. Begin your response with that tool call.`
|
||||
: 'For this turn you must call one of the available tools. Begin your response with the tool call.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- thinking ----------------------------------------------------
|
||||
if (caps.reasoning !== false && profile.thinking !== 'none') {
|
||||
const effort = req.reasoningEffort;
|
||||
const adaptive: Json = profile.display ? { type: 'adaptive', display: 'summarized' } : { type: 'adaptive' };
|
||||
const outputConfig: Json = {};
|
||||
if (effort === 'none') {
|
||||
if (profile.thinking === 'always') {
|
||||
// Cannot be disabled: the lowest effort is the closest thing to "off".
|
||||
body.thinking = adaptive;
|
||||
outputConfig.effort = 'low';
|
||||
} else if (profile.thinking === 'default_on') {
|
||||
body.thinking = { type: profile.off ?? 'disabled' }; // `between_tools` takes no other field
|
||||
}
|
||||
// opt_in / budget models do not think unless asked.
|
||||
} else if (profile.thinking === 'always' || profile.thinking === 'default_on') {
|
||||
// `display: "summarized"` because these models hide the reasoning text by
|
||||
// default, which would look like a long pause in the UI.
|
||||
body.thinking = adaptive;
|
||||
if (effort) outputConfig.effort = effort;
|
||||
} else if (effort) {
|
||||
// Forced tool use cannot be combined with thinking on the older generations.
|
||||
if (profile.thinking === 'opt_in' && !forcing) body.thinking = adaptive;
|
||||
if (profile.thinking === 'budget' && !forcing) {
|
||||
const budget = Math.min(LEGACY_BUDGET[effort], maxTokens - 1_024); // must be < max_tokens, min 1024
|
||||
if (budget >= 1_024) body.thinking = { type: 'enabled', budget_tokens: budget };
|
||||
}
|
||||
if (profile.effort) outputConfig.effort = effort;
|
||||
}
|
||||
if (Object.keys(outputConfig).length > 0) body.output_config = outputConfig;
|
||||
}
|
||||
|
||||
// --- system prompt -----------------------------------------------
|
||||
// There is no schema-less JSON mode (structured outputs need a schema), and
|
||||
// assistant prefill is rejected by current models, so JSON mode is an instruction.
|
||||
if (req.jsonMode) systemNotes.push('Respond with a single valid JSON object and nothing else — no prose, no Markdown code fences.');
|
||||
const system = [req.system?.trim(), ...systemNotes].filter(Boolean).join('\n\n');
|
||||
if (system) body.system = system;
|
||||
|
||||
body.messages = toWireMessages(req.messages, caps, opts.stripThinking);
|
||||
|
||||
if (opts.promptCache) body.cache_control = { type: 'ephemeral' }; // automatic caching of the reusable prefix
|
||||
if (opts.useFallbacks) body.fallbacks = 'default';
|
||||
return body;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Adapter */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function rootUrl(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1$/, '');
|
||||
}
|
||||
|
||||
function isOfficial(cfg: ProviderConfig): boolean {
|
||||
try {
|
||||
return new URL(rootUrl(cfg)).hostname === 'api.anthropic.com';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig, betas: string[] = []): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an Anthropic API key in Settings to use Claude models.');
|
||||
const h: Record<string, string> = { ...cfg.headers, 'x-api-key': cfg.apiKey, 'anthropic-version': API_VERSION };
|
||||
const userBeta = Object.entries(cfg.headers ?? {}).find(([k]) => k.toLowerCase() === 'anthropic-beta')?.[1];
|
||||
const all = [...(userBeta ? userBeta.split(',').map((s) => s.trim()) : []), ...betas].filter(Boolean);
|
||||
for (const k of Object.keys(h)) if (k.toLowerCase() === 'anthropic-beta') delete h[k];
|
||||
if (all.length > 0) h['anthropic-beta'] = [...new Set(all)].join(',');
|
||||
return h;
|
||||
}
|
||||
|
||||
function mapStopReason(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'end_turn':
|
||||
case 'stop_sequence':
|
||||
return 'end';
|
||||
case 'tool_use':
|
||||
return 'tool_use';
|
||||
case 'max_tokens':
|
||||
case 'model_context_window_exceeded':
|
||||
return 'max_tokens';
|
||||
case 'refusal':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other'; // pause_turn (server tools only) and anything new
|
||||
}
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const profile = claudeProfile(req.model);
|
||||
const official = isOfficial(cfg);
|
||||
const opts: BuildOptions = {
|
||||
official,
|
||||
stripThinking: false,
|
||||
// The beta-gated extras are only sent to api.anthropic.com; gateways and
|
||||
// proxies frequently reject fields they do not know.
|
||||
useFallbacks: official && profile.fallbacks,
|
||||
eagerToolInput: official,
|
||||
promptCache: official,
|
||||
};
|
||||
|
||||
// Open the stream, degrading once per recognised 400 (see file header).
|
||||
let http;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
http = await httpRequest(`${rootUrl(cfg)}/v1/messages`, {
|
||||
...ctx,
|
||||
headers: headers(cfg, opts.useFallbacks ? [FALLBACK_BETA] : []),
|
||||
body: buildBody(req, profile, opts),
|
||||
signal: req.signal,
|
||||
});
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
||||
const detail = err.providerMessage ?? '';
|
||||
if (!opts.stripThinking && /signature/i.test(detail) && /thinking/i.test(detail)) opts.stripThinking = true;
|
||||
else if (opts.useFallbacks && /fallback|anthropic-beta/i.test(detail)) opts.useFallbacks = false;
|
||||
else if (opts.eagerToolInput && /eager_input_streaming/i.test(detail)) opts.eagerToolInput = false;
|
||||
else if (opts.promptCache && /cache_control/i.test(detail)) opts.promptCache = false;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Content blocks by index, accumulated exactly as the API defines them.
|
||||
const blocks: Json[] = [];
|
||||
const toolJson = new Map<number, string>();
|
||||
let stopReason = '';
|
||||
let sawStop = false;
|
||||
let reasoningEmitted = false;
|
||||
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
|
||||
const readUsage = (u: unknown) => {
|
||||
if (!isRecord(u)) return;
|
||||
if (typeof u.input_tokens === 'number') usage.input = u.input_tokens;
|
||||
if (typeof u.cache_read_input_tokens === 'number') usage.cacheRead = u.cache_read_input_tokens;
|
||||
if (typeof u.cache_creation_input_tokens === 'number') usage.cacheWrite = u.cache_creation_input_tokens;
|
||||
if (typeof u.output_tokens === 'number') usage.output = u.output_tokens;
|
||||
};
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
switch (data.type) {
|
||||
case 'message_start':
|
||||
if (isRecord(data.message)) readUsage(data.message.usage);
|
||||
break;
|
||||
case 'content_block_start': {
|
||||
const index = num(data.index);
|
||||
const block: Json = isRecord(data.content_block) ? { ...data.content_block } : { type: 'unknown' };
|
||||
blocks[index] = block;
|
||||
if (block.type === 'tool_use') toolJson.set(index, '');
|
||||
if (block.type === 'text' && typeof block.text === 'string' && block.text) yield { type: 'text', delta: block.text };
|
||||
if (block.type === 'thinking' && reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
||||
break;
|
||||
}
|
||||
case 'content_block_delta': {
|
||||
const index = num(data.index);
|
||||
const block = blocks[index];
|
||||
const delta = data.delta;
|
||||
if (!block || !isRecord(delta)) break;
|
||||
if (delta.type === 'text_delta') {
|
||||
const text = str(delta.text);
|
||||
block.text = str(block.text) + text;
|
||||
if (text) yield { type: 'text', delta: text };
|
||||
} else if (delta.type === 'thinking_delta') {
|
||||
const text = str(delta.thinking);
|
||||
block.thinking = str(block.thinking) + text;
|
||||
if (text) {
|
||||
reasoningEmitted = true;
|
||||
yield { type: 'reasoning', delta: text };
|
||||
}
|
||||
} else if (delta.type === 'signature_delta') {
|
||||
block.signature = str(block.signature) + str(delta.signature);
|
||||
} else if (delta.type === 'input_json_delta') {
|
||||
toolJson.set(index, (toolJson.get(index) ?? '') + str(delta.partial_json));
|
||||
} else if (delta.type === 'citations_delta') {
|
||||
const list = Array.isArray(block.citations) ? block.citations : [];
|
||||
list.push(delta.citation);
|
||||
block.citations = list;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'message_delta':
|
||||
if (isRecord(data.delta) && typeof data.delta.stop_reason === 'string') stopReason = data.delta.stop_reason;
|
||||
readUsage(data.usage);
|
||||
break;
|
||||
case 'message_stop':
|
||||
sawStop = true;
|
||||
break;
|
||||
case 'error':
|
||||
// e.g. overloaded_error delivered after the 200 response started
|
||||
throw errorFromProvider(ctx, undefined, data);
|
||||
default:
|
||||
break; // ping, and event types added later
|
||||
}
|
||||
}
|
||||
if (!sawStop && !stopReason) {
|
||||
throw new AIError('network', 'The connection to Anthropic closed before the response finished.', { retryable: true });
|
||||
}
|
||||
|
||||
// Tool calls are surfaced only now, at the end of the message, because the
|
||||
// stop reason decides whether they are usable:
|
||||
// - `refusal` can cut a tool_use off mid-input — never run that turn's tools;
|
||||
// - `max_tokens` can truncate the last call — drop it rather than "repair" it;
|
||||
// - after a mid-output fallback, tool_use blocks from before the last
|
||||
// `fallback` block belong to the declined attempt and must not be echoed.
|
||||
let boundary = -1;
|
||||
blocks.forEach((b, i) => {
|
||||
if (b?.type === 'fallback') boundary = i;
|
||||
});
|
||||
|
||||
const content: ContentPart[] = [];
|
||||
const replay: Json[] = [];
|
||||
const calls: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
if (!block) continue;
|
||||
const beforeBoundary = i < boundary;
|
||||
if (block.type === 'text') {
|
||||
const text = str(block.text);
|
||||
if (!text) continue; // empty text blocks are rejected when echoed back
|
||||
content.push({ type: 'text', text });
|
||||
replay.push(block);
|
||||
} else if (block.type === 'tool_use') {
|
||||
if (beforeBoundary || stopReason === 'refusal') continue;
|
||||
const raw = toolJson.get(i) ?? '';
|
||||
const name = str(block.name);
|
||||
let input = parseArgsStrict(raw);
|
||||
if (!input) {
|
||||
if (stopReason === 'max_tokens') continue; // truncated mid-input
|
||||
input = parseToolArguments(raw, LABEL, name);
|
||||
}
|
||||
const call = { id: str(block.id), name, input };
|
||||
calls.push(call);
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
replay.push({ ...block, input });
|
||||
} else if (block.type === 'fallback') {
|
||||
replay.push(block); // must stay exactly where it appeared
|
||||
} else if (!beforeBoundary) {
|
||||
replay.push(block); // thinking, redacted_thinking, …: echoed back unchanged
|
||||
}
|
||||
}
|
||||
|
||||
for (const call of calls) yield { type: 'tool_call', call };
|
||||
|
||||
let mapped = mapStopReason(stopReason);
|
||||
if (mapped === 'tool_use' && calls.length === 0) mapped = 'other';
|
||||
if (mapped === 'end' && calls.length > 0) mapped = 'tool_use';
|
||||
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
if (replay.length > 0) message.providerData = { kind: 'anthropic', data: { content: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: mapped,
|
||||
// Anthropic reports uncached, cache-read and cache-write input separately.
|
||||
usage: makeUsage({
|
||||
input: usage.input + usage.cacheRead + usage.cacheWrite,
|
||||
output: usage.output,
|
||||
cacheRead: usage.cacheRead,
|
||||
cacheWrite: usage.cacheWrite,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function supported(caps: unknown, key: string): boolean | undefined {
|
||||
if (!isRecord(caps) || !isRecord(caps[key])) return undefined;
|
||||
const v = (caps[key] as Json).supported;
|
||||
return typeof v === 'boolean' ? v : undefined;
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const ctx = errorContext(cfg);
|
||||
const found: DiscoveredModel[] = [];
|
||||
let afterId = '';
|
||||
// https://platform.claude.com/docs/en/api/models/list — cursor pagination via has_more / last_id.
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const url = `${rootUrl(cfg)}/v1/models?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ''}`;
|
||||
const res = await requestJson<Json>(url, { ...ctx, headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
||||
for (const m of data) {
|
||||
const id = str(m.id);
|
||||
if (!id || !isChatModelId(id)) continue;
|
||||
const model: DiscoveredModel = {
|
||||
id,
|
||||
// The API version in use rejects sampling parameters on every model.
|
||||
capabilities: { tools: true, temperature: false },
|
||||
};
|
||||
if (typeof m.display_name === 'string') model.label = m.display_name;
|
||||
if (typeof m.max_input_tokens === 'number') model.contextWindow = m.max_input_tokens;
|
||||
if (typeof m.max_tokens === 'number') model.maxOutput = m.max_tokens;
|
||||
const vision = supported(m.capabilities, 'image_input');
|
||||
const pdf = supported(m.capabilities, 'pdf_input');
|
||||
const thinking = supported(m.capabilities, 'thinking');
|
||||
if (vision !== undefined) model.capabilities = { ...model.capabilities, vision };
|
||||
if (pdf !== undefined) model.capabilities = { ...model.capabilities, pdf };
|
||||
if (thinking !== undefined) model.capabilities = { ...model.capabilities, reasoning: thinking };
|
||||
found.push(model);
|
||||
}
|
||||
if (res.has_more !== true || typeof res.last_id !== 'string' || !res.last_id) break;
|
||||
afterId = res.last_id;
|
||||
}
|
||||
return withCatalogDefaults('anthropic', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
// Listing models is free and proves both connectivity and the key.
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const anthropicAdapter: ProviderAdapter = {
|
||||
kind: 'anthropic',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://platform.claude.com/settings/keys',
|
||||
keyHint: 'sk-ant-…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
206
server/ai/providers/catalog.ts
Normal file
206
server/ai/providers/catalog.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* Static defaults for well-known models: capabilities, limits and list prices.
|
||||
*
|
||||
* Discovery APIs return ids (and sometimes limits) but almost never prices or
|
||||
* feature flags, so `listModels` merges its results with this table. Unknown
|
||||
* models simply get no defaults — nothing here is required for a model to work.
|
||||
*
|
||||
* Everything below was read from the official pages listed per provider on the
|
||||
* dates noted. Prices are USD per million tokens, standard (non-batch,
|
||||
* non-priority) tier, short-context tier where a provider has several. A field
|
||||
* is omitted when it could not be confirmed from the page — do not guess.
|
||||
*/
|
||||
import type { DiscoveredModel, ModelCapabilities, ProviderKind } from '../types.ts';
|
||||
|
||||
export const CATALOG_UPDATED = '2026-11-10';
|
||||
|
||||
type Entry = Omit<DiscoveredModel, 'id' | 'capabilities'> & { capabilities?: Partial<ModelCapabilities> };
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI */
|
||||
/* Prices: https://developers.openai.com/api/docs/pricing (fetched 2026-11-11)
|
||||
/* Limits: https://developers.openai.com/api/docs/models (fetched 2026-11-11;
|
||||
/* only the four current flagship models list limits there — the
|
||||
/* limits on older models below are the long-published values and were
|
||||
/* not re-fetched)
|
||||
/* Reasoning models reject a custom `temperature`, hence temperature:false.
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const OA_REASONING: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
||||
const OA_CLASSIC: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: false, temperature: true };
|
||||
const OA_FLAGSHIP_LIMITS = { contextWindow: 1_050_000, maxOutput: 128_000 };
|
||||
|
||||
const OPENAI: Record<string, Entry> = {
|
||||
'gpt-6-astra': { label: 'GPT-6 Astra', ...OA_FLAGSHIP_LIMITS, priceIn: 10, priceOut: 50, capabilities: OA_REASONING },
|
||||
'gpt-6.1-sol': { label: 'GPT-6.1 Sol', ...OA_FLAGSHIP_LIMITS, priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-6.1-terra': { label: 'GPT-6.1 Terra', ...OA_FLAGSHIP_LIMITS, priceIn: 1, priceOut: 5, capabilities: OA_REASONING },
|
||||
'gpt-6-luna': { label: 'GPT-6 Luna', ...OA_FLAGSHIP_LIMITS, priceIn: 0.1, priceOut: 0.5, capabilities: OA_REASONING },
|
||||
'gpt-6-sol': { label: 'GPT-6 Sol', priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5.6-sol': { label: 'GPT-5.6 Sol', priceIn: 4, priceOut: 20, capabilities: OA_REASONING },
|
||||
'gpt-5.6-terra': { label: 'GPT-5.6 Terra', priceIn: 2, priceOut: 12, capabilities: OA_REASONING },
|
||||
'gpt-5.6-luna': { label: 'GPT-5.6 Luna', priceIn: 0.2, priceOut: 1.2, capabilities: OA_REASONING },
|
||||
'gpt-5.5': { label: 'GPT-5.5', priceIn: 5, priceOut: 30, capabilities: OA_REASONING },
|
||||
'gpt-5.5-pro': { label: 'GPT-5.5 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
||||
'gpt-5.4': { label: 'GPT-5.4', priceIn: 2.5, priceOut: 15, capabilities: OA_REASONING },
|
||||
'gpt-5.4-pro': { label: 'GPT-5.4 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
||||
'gpt-5.4-mini': { label: 'GPT-5.4 mini', priceIn: 0.75, priceOut: 4.5, capabilities: OA_REASONING },
|
||||
'gpt-5.4-nano': { label: 'GPT-5.4 nano', priceIn: 0.2, priceOut: 1.25, capabilities: OA_REASONING },
|
||||
'gpt-5.3-codex': { label: 'GPT-5.3 Codex', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
||||
'gpt-5.2': { label: 'GPT-5.2', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
||||
'gpt-5.2-pro': { label: 'GPT-5.2 Pro', priceIn: 21, priceOut: 168, capabilities: OA_REASONING },
|
||||
'gpt-5.1': { label: 'GPT-5.1', priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5': { label: 'GPT-5', contextWindow: 400_000, maxOutput: 128_000, priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5-mini': { label: 'GPT-5 mini', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.25, priceOut: 2, capabilities: OA_REASONING },
|
||||
'gpt-5-nano': { label: 'GPT-5 nano', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.05, priceOut: 0.4, capabilities: OA_REASONING },
|
||||
'gpt-5-pro': { label: 'GPT-5 Pro', priceIn: 15, priceOut: 120, capabilities: OA_REASONING },
|
||||
'chat-latest': { label: 'ChatGPT (latest)', priceIn: 5, priceOut: 30, capabilities: OA_CLASSIC },
|
||||
'gpt-4.1': { label: 'GPT-4.1', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 2, priceOut: 8, capabilities: OA_CLASSIC },
|
||||
'gpt-4.1-mini': { label: 'GPT-4.1 mini', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 0.4, priceOut: 1.6, capabilities: OA_CLASSIC },
|
||||
'gpt-4o': { label: 'GPT-4o', contextWindow: 128_000, maxOutput: 16_384, priceIn: 2.5, priceOut: 10, capabilities: OA_CLASSIC },
|
||||
'gpt-4o-mini': { label: 'GPT-4o mini', contextWindow: 128_000, maxOutput: 16_384, priceIn: 0.15, priceOut: 0.6, capabilities: OA_CLASSIC },
|
||||
o3: { label: 'o3', contextWindow: 200_000, maxOutput: 100_000, priceIn: 2, priceOut: 8, capabilities: OA_REASONING },
|
||||
'o3-pro': { label: 'o3-pro', contextWindow: 200_000, maxOutput: 100_000, priceIn: 20, priceOut: 80, capabilities: OA_REASONING },
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Anthropic */
|
||||
/* Prices: https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-11-11)
|
||||
/* Ids and limits: https://platform.claude.com/docs/en/about-claude/models/overview
|
||||
/* (via Anthropic's Claude API reference, cached 2026-11-03)
|
||||
/* temperature:false for every Claude model: the adapter speaks API version
|
||||
/* 2026-10-21, which rejects temperature/top_p/top_k on all models
|
||||
/* (https://platform.claude.com/docs/en/api/migration-2026-10-21).
|
||||
/* Claude Haiku 5.5 prices are for prompts up to 100K tokens ($0.50/$2.50 above).
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const CLAUDE: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
||||
const CLAUDE_1M = { contextWindow: 1_000_000, maxOutput: 128_000 };
|
||||
|
||||
const ANTHROPIC: Record<string, Entry> = {
|
||||
'claude-fable-5-2': { label: 'Claude Fable 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
||||
'claude-mythos-5-2': { label: 'Claude Mythos 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
||||
'claude-fable-5-1': { label: 'Claude Fable 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-mythos-5-1': { label: 'Claude Mythos 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-fable-5': { label: 'Claude Fable 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-mythos-5': { label: 'Claude Mythos 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-opus-5-5': { label: 'Claude Opus 5.5', ...CLAUDE_1M, priceIn: 4, priceOut: 20, capabilities: CLAUDE },
|
||||
'claude-opus-5': { label: 'Claude Opus 5', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-8': { label: 'Claude Opus 4.8', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-7': { label: 'Claude Opus 4.7', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-6': { label: 'Claude Opus 4.6', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-5': { label: 'Claude Opus 4.5', priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-sonnet-5-5': { label: 'Claude Sonnet 5.5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
||||
'claude-sonnet-5': { label: 'Claude Sonnet 5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
||||
'claude-sonnet-4-6': { label: 'Claude Sonnet 4.6', ...CLAUDE_1M, priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
||||
'claude-sonnet-4-5': { label: 'Claude Sonnet 4.5', priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
||||
'claude-haiku-5-5': { label: 'Claude Haiku 5.5', ...CLAUDE_1M, priceIn: 0.1, priceOut: 0.5, capabilities: CLAUDE },
|
||||
'claude-haiku-4-5': { label: 'Claude Haiku 4.5', contextWindow: 200_000, maxOutput: 64_000, priceIn: 1, priceOut: 5, capabilities: CLAUDE },
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Google Gemini */
|
||||
/* Prices: https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-11-11,
|
||||
/* now served from https://aistudio.google.com/docs/pricing)
|
||||
/* Limits: https://ai.google.dev/gemini-api/docs/models (fetched 2026-11-11;
|
||||
/* only the Gemini 4 models list limits — `GET /v1beta/models`
|
||||
/* supplies inputTokenLimit/outputTokenLimit for the rest at runtime)
|
||||
/* Scheduled changes announced on the pricing page (NOT applied here):
|
||||
/* - gemini-4-carbon / gemini-4-argon: $4 / $20 from 2026-12-01
|
||||
/* - gemini-3.8-flash / gemini-3.6-flash: $1.50 / $7.50 from 2027-01-01
|
||||
/* - gemini-2.5-pro / gemini-2.5-flash are deprecated and are routed to
|
||||
/* gemini-3.8-flash / gemini-3.5-flash-lite (and billed as such) from 2026-12-01
|
||||
/* Pro-tier prices are for prompts up to 200K tokens.
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const GEMINI_CAPS: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: true };
|
||||
const GEMINI_4_LIMITS = { contextWindow: 1_048_576, maxOutput: 65_536 };
|
||||
|
||||
const GEMINI: Record<string, Entry> = {
|
||||
'gemini-4-carbon': { label: 'Gemini 4 Carbon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-4-argon': { label: 'Gemini 4 Argon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.8-flash': { label: 'Gemini 3.8 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.6-flash': { label: 'Gemini 3.6 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.5-flash-lite': { label: 'Gemini 3.5 Flash-Lite', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.1-pro-preview': { label: 'Gemini 3.1 Pro (preview)', priceIn: 2, priceOut: 12, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.1-flash-lite': { label: 'Gemini 3.1 Flash-Lite', priceIn: 0.25, priceOut: 1.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-3-flash-preview': { label: 'Gemini 3 Flash (preview)', priceIn: 0.5, priceOut: 3, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-pro': { label: 'Gemini 2.5 Pro', priceIn: 1.25, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-flash': { label: 'Gemini 2.5 Flash', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-flash-lite': { label: 'Gemini 2.5 Flash-Lite', priceIn: 0.1, priceOut: 0.4, capabilities: GEMINI_CAPS },
|
||||
};
|
||||
|
||||
const TABLES: Record<ProviderKind, Record<string, Entry>> = {
|
||||
openai: OPENAI,
|
||||
anthropic: ANTHROPIC,
|
||||
gemini: GEMINI,
|
||||
// Aggregators and local servers host arbitrary models at their own prices.
|
||||
openai_compatible: {},
|
||||
ollama: {},
|
||||
};
|
||||
|
||||
/**
|
||||
* What may follow a known id for it to still be "the same model at the same
|
||||
* price": a dated snapshot or a moving alias, e.g. `gpt-5-2025-08-07`,
|
||||
* `claude-haiku-4-5-20251001`, `claude-opus-4-5@20251101`, `gemini-2.5-flash-latest`.
|
||||
*/
|
||||
const SNAPSHOT_SUFFIX = /^[-@:](\d{4}-\d{2}-\d{2}|\d{6,8}|\d{3,4}|latest|preview|preview-\d{2}-\d{2,4}|v\d+(:\d+)?)$/;
|
||||
|
||||
function clone(e: Entry): Partial<DiscoveredModel> {
|
||||
return { ...e, ...(e.capabilities ? { capabilities: { ...e.capabilities } } : {}) };
|
||||
}
|
||||
|
||||
function lookupIn(table: Record<string, Entry>, id: string): Partial<DiscoveredModel> {
|
||||
const exact = table[id];
|
||||
if (exact) return clone(exact);
|
||||
|
||||
// Longest known id that is a prefix of `id` and ends on a separator.
|
||||
let best = '';
|
||||
for (const key of Object.keys(table)) {
|
||||
if (key.length > best.length && id.startsWith(key) && /[-@:]/.test(id.charAt(key.length))) best = key;
|
||||
}
|
||||
if (!best) return {};
|
||||
const entry = table[best];
|
||||
if (!entry) return {};
|
||||
const out = clone(entry);
|
||||
delete out.label; // the label names the base model, not this variant
|
||||
if (!SNAPSHOT_SUFFIX.test(id.slice(best.length))) {
|
||||
// A genuinely different variant (`o3-deep-research`, `gpt-4o-audio-...`):
|
||||
// capabilities are a fair guess, the price is not.
|
||||
delete out.priceIn;
|
||||
delete out.priceOut;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Defaults for a model: exact id first, then the longest known id that prefixes
|
||||
* it (so dated snapshots match their base model). Returns `{}` when unknown.
|
||||
*/
|
||||
export function lookupModelDefaults(kind: ProviderKind, modelId: string): Partial<DiscoveredModel> {
|
||||
// Gemini's listing names models "models/<id>".
|
||||
const id = modelId.trim().replace(/^models\//, '');
|
||||
if (!id) return {};
|
||||
|
||||
if (kind === 'openai_compatible') {
|
||||
// Routers name models "<vendor>/<model>" (`anthropic/claude-sonnet-5-5`).
|
||||
// Borrow capabilities and limits from the vendor's table, never its prices.
|
||||
const bare = id.slice(id.lastIndexOf('/') + 1).replace(/:[a-z]+$/, '');
|
||||
for (const table of [OPENAI, ANTHROPIC, GEMINI]) {
|
||||
const hit = lookupIn(table, bare);
|
||||
if (Object.keys(hit).length > 0) {
|
||||
delete hit.priceIn;
|
||||
delete hit.priceOut;
|
||||
// Temperature limits are properties of the first-party APIs, not of the router.
|
||||
if (hit.capabilities) delete hit.capabilities.temperature;
|
||||
return hit;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
return lookupIn(TABLES[kind], id);
|
||||
}
|
||||
|
||||
/** Every catalogued model id for a provider (used as a fallback list when discovery is unavailable). */
|
||||
export function catalogModelIds(kind: ProviderKind): string[] {
|
||||
return Object.keys(TABLES[kind]);
|
||||
}
|
||||
138
server/ai/providers/common.ts
Normal file
138
server/ai/providers/common.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
/**
|
||||
* Helpers shared by the adapters that are about our neutral contract rather
|
||||
* than about HTTP: tool-argument parsing, model-list post-processing, the
|
||||
* connection-test wrapper.
|
||||
*/
|
||||
import { jsonrepair } from 'jsonrepair';
|
||||
import { AIError, type DiscoveredModel, type ProviderKind, type TestResult, type Usage } from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { isRecord } from './http.ts';
|
||||
|
||||
/** Strict parse of tool-call arguments; `undefined` when they are not a JSON object. */
|
||||
export function parseArgsStrict(raw: string): Record<string, unknown> | undefined {
|
||||
const text = raw.trim();
|
||||
if (text === '') return {};
|
||||
try {
|
||||
const value: unknown = JSON.parse(text);
|
||||
return isRecord(value) ? value : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the JSON arguments of a completed tool call. Models occasionally emit
|
||||
* slightly broken JSON (trailing commas, unescaped newlines); `jsonrepair`
|
||||
* fixes those. Anything still unparseable is an `invalid_response`.
|
||||
*
|
||||
* Do NOT use this on a call that was cut off by the output-token limit —
|
||||
* repairing a truncated call would silently invent a shorter input.
|
||||
*/
|
||||
export function parseToolArguments(raw: unknown, provider: string, toolName: string): Record<string, unknown> {
|
||||
if (isRecord(raw)) return raw; // some servers send an object instead of a JSON string
|
||||
const text = typeof raw === 'string' ? raw : raw == null ? '' : String(raw);
|
||||
const strict = parseArgsStrict(text);
|
||||
if (strict) return strict;
|
||||
try {
|
||||
const repaired: unknown = JSON.parse(jsonrepair(text));
|
||||
if (isRecord(repaired)) return repaired;
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, {
|
||||
providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text,
|
||||
});
|
||||
}
|
||||
|
||||
/** Stable id for providers that do not issue tool-call ids themselves. */
|
||||
export function newToolCallId(prefix: string): string {
|
||||
return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`;
|
||||
}
|
||||
|
||||
export function dataUrl(mime: string, base64: string): string {
|
||||
return `data:${mime};base64,${base64}`;
|
||||
}
|
||||
|
||||
export function num(v: unknown): number {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
||||
}
|
||||
|
||||
export function str(v: unknown): string {
|
||||
return typeof v === 'string' ? v : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Usage convention used by every adapter:
|
||||
* inputTokens = ALL prompt tokens, including cached ones
|
||||
* cacheRead/cacheWriteTokens = the cached subsets of inputTokens
|
||||
* outputTokens = ALL generated tokens, including reasoning
|
||||
* reasoningTokens = the reasoning subset of outputTokens
|
||||
*/
|
||||
export function makeUsage(u: { input: number; output: number; cacheRead?: number; cacheWrite?: number; reasoning?: number }): Usage {
|
||||
const usage: Usage = { inputTokens: u.input, outputTokens: u.output };
|
||||
if (u.cacheRead) usage.cacheReadTokens = u.cacheRead;
|
||||
if (u.cacheWrite) usage.cacheWriteTokens = u.cacheWrite;
|
||||
if (u.reasoning) usage.reasoningTokens = u.reasoning;
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids that are obviously not text-chat models: embeddings, speech, image and
|
||||
* video generation, moderation, realtime/audio-only, rerankers, legacy
|
||||
* completion-only models.
|
||||
*/
|
||||
const NON_CHAT_RE = new RegExp(
|
||||
[
|
||||
'embed', // text-embedding-3-large, gemini-embedding-001, nomic-embed-text
|
||||
'\\btts\\b|whisper|transcri|speech|\\baudio\\b|realtime|\\blive\\b',
|
||||
'dall-e|gpt-image|image-generation|imagen|-image\\b|image-preview|\\bsora\\b|\\bveo\\b|lyria|stable-diffusion',
|
||||
'moderation|-guard\\b|rerank|\\baqa\\b',
|
||||
'computer-use|robotics|deep-research',
|
||||
'^(babbage|davinci|curie|ada)\\b|^text-|turbo-instruct',
|
||||
].join('|'),
|
||||
'i',
|
||||
);
|
||||
|
||||
export function isChatModelId(id: string): boolean {
|
||||
return !NON_CHAT_RE.test(id);
|
||||
}
|
||||
|
||||
/** Merges discovery results with catalog defaults; values the provider reported always win. */
|
||||
export function withCatalogDefaults(kind: ProviderKind, models: DiscoveredModel[]): DiscoveredModel[] {
|
||||
const seen = new Set<string>();
|
||||
const out: DiscoveredModel[] = [];
|
||||
for (const m of models) {
|
||||
if (!m.id || seen.has(m.id)) continue;
|
||||
seen.add(m.id);
|
||||
const d = lookupModelDefaults(kind, m.id);
|
||||
const merged: DiscoveredModel = { ...d, ...stripUndefined(m), id: m.id };
|
||||
if (d.capabilities || m.capabilities) merged.capabilities = { ...d.capabilities, ...stripUndefined(m.capabilities ?? {}) };
|
||||
out.push(merged);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function stripUndefined<T extends object>(obj: T): T {
|
||||
const out = {} as Record<string, unknown>;
|
||||
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null) out[k] = v;
|
||||
return out as T;
|
||||
}
|
||||
|
||||
/** Shared implementation of `ProviderAdapter.test()`: never throws, always friendly. */
|
||||
export async function runTest(label: string, check: () => Promise<{ modelCount?: number; note?: string }>): Promise<TestResult> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const { modelCount, note } = await check();
|
||||
const latencyMs = Date.now() - started;
|
||||
const message =
|
||||
note ?? (modelCount !== undefined ? `Connected to ${label} — ${modelCount} model${modelCount === 1 ? '' : 's'} available.` : `Connected to ${label}.`);
|
||||
return modelCount !== undefined ? { ok: true, message, latencyMs, modelCount } : { ok: true, message, latencyMs };
|
||||
} catch (err) {
|
||||
const latencyMs = Date.now() - started;
|
||||
const message = err instanceof AIError ? err.message : `Could not connect to ${label}.`;
|
||||
return { ok: false, message, latencyMs };
|
||||
}
|
||||
}
|
||||
|
||||
/** Timeout used by `test()` so a dead endpoint fails fast instead of hanging the settings page. */
|
||||
export const TEST_TIMEOUT_MS = 20_000;
|
||||
342
server/ai/providers/gemini.ts
Normal file
342
server/ai/providers/gemini.ts
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
/**
|
||||
* Google Gemini — `models/{model}:streamGenerateContent?alt=sse`.
|
||||
* https://ai.google.dev/api/generate-content
|
||||
* https://ai.google.dev/gemini-api/docs/function-calling
|
||||
* https://ai.google.dev/gemini-api/docs/thinking
|
||||
* https://ai.google.dev/gemini-api/docs/thought-signatures
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - The API key travels in the `x-goog-api-key` header, never in the URL.
|
||||
* - Function declarations accept only a subset of JSON Schema, so schemas are
|
||||
* down-converted (`toGeminiSchema`).
|
||||
* - Gemini issues no tool-call ids; stable ones are synthesised.
|
||||
* - Thinking models attach a `thoughtSignature` to parts (notably function calls)
|
||||
* that must be sent back unchanged, so each turn's raw parts live in `providerData`.
|
||||
* - Thinking is configured with `thinkingLevel` on Gemini 3 and later and with a
|
||||
* numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried.
|
||||
* - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY).
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { isChatModelId, makeUsage, newToolCallId, num, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'Google Gemini';
|
||||
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
|
||||
|
||||
function root(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1(beta)?$/, '');
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add a Gemini API key in Settings to use Gemini models.');
|
||||
return { ...cfg.headers, 'x-goog-api-key': cfg.apiKey };
|
||||
}
|
||||
|
||||
const SCHEMA_KEYS = new Set(['type', 'description', 'enum', 'properties', 'required', 'items', 'nullable', 'format', 'minimum', 'maximum', 'minItems', 'maxItems', 'anyOf', 'title']);
|
||||
const SCHEMA_FORMATS = new Set(['date-time', 'enum', 'int32', 'int64', 'float', 'double']);
|
||||
|
||||
/** Reduces JSON Schema to the OpenAPI-style subset Gemini's `parameters` field accepts. */
|
||||
export function toGeminiSchema(schema: unknown): Json {
|
||||
if (!isRecord(schema)) return { type: 'string' };
|
||||
const out: Json = {};
|
||||
for (const [key, value] of Object.entries(schema)) {
|
||||
if (!SCHEMA_KEYS.has(key)) continue; // additionalProperties, $schema, default, pattern, const, ...
|
||||
if (key === 'properties' && isRecord(value)) {
|
||||
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
|
||||
} else if (key === 'items') {
|
||||
out.items = toGeminiSchema(value);
|
||||
} else if (key === 'anyOf' && Array.isArray(value)) {
|
||||
out.anyOf = value.map(toGeminiSchema);
|
||||
} else if (key === 'type' && Array.isArray(value)) {
|
||||
// ["string","null"] -> nullable string
|
||||
const real = value.filter((t) => t !== 'null');
|
||||
out.type = real[0] ?? 'string';
|
||||
if (value.includes('null')) out.nullable = true;
|
||||
} else if (key === 'format') {
|
||||
if (typeof value === 'string' && SCHEMA_FORMATS.has(value)) out.format = value;
|
||||
} else {
|
||||
out[key] = value;
|
||||
}
|
||||
}
|
||||
if (out.enum && !out.type) out.type = 'string';
|
||||
if (out.type === 'object' && !out.properties) out.properties = {};
|
||||
return out;
|
||||
}
|
||||
|
||||
function userParts(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'tool_result') {
|
||||
out.push({ functionResponse: { name: p.name, response: p.isError ? { error: p.output } : { result: p.output } } });
|
||||
} else if (p.type === 'text' && p.text) {
|
||||
out.push({ text: p.text });
|
||||
} else if (p.type === 'image') {
|
||||
if (caps.vision === false) out.push({ text: '[An image was attached, but this model cannot read images.]' });
|
||||
else out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
||||
else if (p.mime.startsWith('text/')) out.push({ text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else out.push({ text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function toContents(messages: ChatMessage[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const contents: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
let parts: Json[];
|
||||
if (msg.role === 'assistant') {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'gemini' && isRecord(pd.data) && Array.isArray(pd.data.parts)) {
|
||||
parts = pd.data.parts.filter(isRecord); // thought signatures included, exactly as received
|
||||
} else {
|
||||
parts = [];
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'text' && p.text) parts.push({ text: p.text });
|
||||
else if (p.type === 'tool_call') parts.push({ functionCall: { name: p.name, args: p.input ?? {} } });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
parts = userParts(msg.content, caps);
|
||||
}
|
||||
if (parts.length === 0) continue;
|
||||
const role = msg.role === 'assistant' ? 'model' : 'user';
|
||||
const last = contents[contents.length - 1];
|
||||
// Gemini requires strictly alternating roles; merge consecutive turns of the same role.
|
||||
if (last && last.role === role) (last.parts as Json[]).push(...parts);
|
||||
else contents.push({ role, parts });
|
||||
}
|
||||
if (contents[0]?.role === 'model') contents.unshift({ role: 'user', parts: [{ text: '(continuing the conversation)' }] });
|
||||
return contents;
|
||||
}
|
||||
|
||||
type ThinkingStyle = 'level' | 'budget' | 'off';
|
||||
|
||||
function thinkingStyleFor(model: string): ThinkingStyle {
|
||||
const m = /gemini-(\d+)(?:\.(\d+))?/.exec(model);
|
||||
if (!m) return 'level';
|
||||
const major = Number(m[1]);
|
||||
if (major >= 3) return 'level';
|
||||
if (major === 2 && Number(m[2] ?? 0) >= 5) return 'budget';
|
||||
return 'off';
|
||||
}
|
||||
|
||||
const BUDGETS = { none: 0, low: 2_048, medium: 8_192, high: 24_576 } as const;
|
||||
|
||||
function buildBody(req: ChatRequest, thinking: ThinkingStyle, jsonMode: boolean): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { contents: toContents(req.messages, caps) };
|
||||
if (req.system?.trim()) body.systemInstruction = { parts: [{ text: req.system.trim() }] };
|
||||
|
||||
const hasTools = caps.tools !== false && (req.tools?.length ?? 0) > 0;
|
||||
if (hasTools) {
|
||||
body.tools = [{ functionDeclarations: req.tools!.map((t) => ({ name: t.name, description: t.description, parameters: toGeminiSchema({ type: 'object', ...t.parameters }) })) }];
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none') body.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
|
||||
else if (choice === 'required') body.toolConfig = { functionCallingConfig: { mode: 'ANY' } };
|
||||
else if (isRecord(choice)) body.toolConfig = { functionCallingConfig: { mode: 'ANY', allowedFunctionNames: [choice.name] } };
|
||||
}
|
||||
|
||||
const gen: Json = {};
|
||||
if (req.temperature !== undefined && caps.temperature !== false) gen.temperature = req.temperature;
|
||||
if (req.maxOutputTokens) gen.maxOutputTokens = req.maxOutputTokens;
|
||||
// JSON mode cannot be combined with function calling on most models.
|
||||
if (jsonMode && req.jsonMode && !hasTools) gen.responseMimeType = 'application/json';
|
||||
if (caps.reasoning !== false && thinking !== 'off') {
|
||||
const effort = req.reasoningEffort;
|
||||
if (thinking === 'level') {
|
||||
const cfg: Json = { includeThoughts: true };
|
||||
if (effort) cfg.thinkingLevel = effort === 'none' ? 'minimal' : effort;
|
||||
gen.thinkingConfig = cfg;
|
||||
} else if (effort) {
|
||||
// 2.5 Pro cannot switch thinking off; its minimum budget is 128 tokens.
|
||||
const budget = effort === 'none' && /pro/.test(req.model) ? 128 : BUDGETS[effort];
|
||||
gen.thinkingConfig = { thinkingBudget: budget, includeThoughts: budget > 0 };
|
||||
} else {
|
||||
gen.thinkingConfig = { includeThoughts: true };
|
||||
}
|
||||
}
|
||||
if (Object.keys(gen).length > 0) body.generationConfig = gen;
|
||||
return body;
|
||||
}
|
||||
|
||||
function mapFinish(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'STOP':
|
||||
return 'end';
|
||||
case 'MAX_TOKENS':
|
||||
return 'max_tokens';
|
||||
case 'SAFETY':
|
||||
case 'RECITATION':
|
||||
case 'BLOCKLIST':
|
||||
case 'PROHIBITED_CONTENT':
|
||||
case 'SPII':
|
||||
case 'IMAGE_SAFETY':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other'; // MALFORMED_FUNCTION_CALL, OTHER, UNEXPECTED_TOOL_CALL, ...
|
||||
}
|
||||
}
|
||||
|
||||
function modelPath(model: string): string {
|
||||
return model.startsWith('models/') || model.startsWith('tunedModels/') ? model : `models/${model}`;
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const url = `${root(cfg)}/v1beta/${modelPath(req.model)}:streamGenerateContent?alt=sse`;
|
||||
let thinking = thinkingStyleFor(req.model);
|
||||
let jsonMode = true;
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(url, { ...ctx, headers: headers(cfg), body: buildBody(req, thinking, jsonMode), signal: req.signal });
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (thinking !== 'off' && /thinking|thought/.test(detail)) thinking = thinking === 'level' ? 'budget' : 'off';
|
||||
else if (jsonMode && req.jsonMode && /response_?mime|json/.test(detail)) jsonMode = false;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const parts: Json[] = [];
|
||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
let text = '';
|
||||
let finish = '';
|
||||
let blocked = '';
|
||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
||||
if (isRecord(data.promptFeedback) && str(data.promptFeedback.blockReason)) blocked = str(data.promptFeedback.blockReason);
|
||||
if (isRecord(data.usageMetadata)) {
|
||||
const u = data.usageMetadata;
|
||||
usage.input = num(u.promptTokenCount) || usage.input;
|
||||
usage.reasoning = num(u.thoughtsTokenCount) || usage.reasoning;
|
||||
// candidatesTokenCount excludes thinking; both are billed as output.
|
||||
usage.output = num(u.candidatesTokenCount) + num(u.thoughtsTokenCount) || usage.output;
|
||||
usage.cacheRead = num(u.cachedContentTokenCount) || usage.cacheRead;
|
||||
}
|
||||
const candidate = Array.isArray(data.candidates) ? data.candidates[0] : undefined;
|
||||
if (!isRecord(candidate)) continue;
|
||||
if (str(candidate.finishReason)) finish = str(candidate.finishReason);
|
||||
const chunkParts = isRecord(candidate.content) && Array.isArray(candidate.content.parts) ? candidate.content.parts.filter(isRecord) : [];
|
||||
for (const part of chunkParts) {
|
||||
if (isRecord(part.functionCall)) {
|
||||
const call = { id: str(part.functionCall.id) || newToolCallId('gem'), name: str(part.functionCall.name), input: isRecord(part.functionCall.args) ? part.functionCall.args : {} };
|
||||
calls.push(call);
|
||||
parts.push(part);
|
||||
yield { type: 'tool_call', call };
|
||||
} else if (typeof part.text === 'string') {
|
||||
if (part.thought === true) {
|
||||
if (part.text) yield { type: 'reasoning', delta: part.text };
|
||||
// Thought summaries are display-only; only a signature needs to survive.
|
||||
if (part.thoughtSignature) parts.push({ text: '', thought: true, thoughtSignature: part.thoughtSignature });
|
||||
} else {
|
||||
if (part.text) {
|
||||
text += part.text;
|
||||
yield { type: 'text', delta: part.text };
|
||||
}
|
||||
// Merge streamed text fragments, but keep a part that carries a signature intact.
|
||||
const last = parts[parts.length - 1];
|
||||
if (last && typeof last.text === 'string' && last.thought !== true && !last.thoughtSignature && !part.thoughtSignature) last.text += part.text;
|
||||
else parts.push({ ...part });
|
||||
}
|
||||
} else {
|
||||
parts.push(part); // inlineData, executableCode, ... replayed untouched
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blocked && !text && calls.length === 0) {
|
||||
throw new AIError('content_filter', 'Gemini blocked this request under its safety policy.', { providerMessage: `blockReason: ${blocked}` });
|
||||
}
|
||||
if (!finish && !text && calls.length === 0) {
|
||||
throw new AIError('network', 'The connection to Gemini closed before the response finished.', { retryable: true });
|
||||
}
|
||||
if (finish === 'MALFORMED_FUNCTION_CALL' && calls.length === 0) {
|
||||
throw new AIError('invalid_response', 'Gemini produced a malformed tool call. Try again.', { retryable: true });
|
||||
}
|
||||
|
||||
const content: ContentPart[] = [];
|
||||
if (text) content.push({ type: 'text', text });
|
||||
for (const call of calls) content.push({ type: 'tool_call', ...call });
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
const replay = parts.filter((p) => !(typeof p.text === 'string' && p.text === '' && !p.thoughtSignature));
|
||||
if (replay.length > 0) message.providerData = { kind: 'gemini', data: { parts: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: calls.length > 0 ? 'tool_use' : mapFinish(finish || 'STOP'),
|
||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
||||
};
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const found: DiscoveredModel[] = [];
|
||||
let pageToken = '';
|
||||
for (let page = 0; page < 10; page += 1) {
|
||||
const url = `${root(cfg)}/v1beta/models?pageSize=1000${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
|
||||
const res = await requestJson<Json>(url, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
for (const m of Array.isArray(res.models) ? res.models.filter(isRecord) : []) {
|
||||
const id = str(m.name).replace(/^models\//, '');
|
||||
const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods.map(String) : [];
|
||||
if (!id || !methods.includes('generateContent') || !isChatModelId(id)) continue;
|
||||
if (!/^(gemini|gemma|learnlm)/.test(id)) continue;
|
||||
const model: DiscoveredModel = { id };
|
||||
if (str(m.displayName)) model.label = str(m.displayName);
|
||||
if (num(m.inputTokenLimit)) model.contextWindow = num(m.inputTokenLimit);
|
||||
if (num(m.outputTokenLimit)) model.maxOutput = num(m.outputTokenLimit);
|
||||
if (typeof m.thinking === 'boolean') model.capabilities = { reasoning: m.thinking };
|
||||
if (id.startsWith('gemma')) model.capabilities = { ...model.capabilities, tools: false, pdf: false };
|
||||
found.push(model);
|
||||
}
|
||||
pageToken = str(res.nextPageToken);
|
||||
if (!pageToken) break;
|
||||
}
|
||||
return withCatalogDefaults('gemini', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const geminiAdapter: ProviderAdapter = {
|
||||
kind: 'gemini',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://aistudio.google.com/apikey',
|
||||
keyHint: 'AIza…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
519
server/ai/providers/http.ts
Normal file
519
server/ai/providers/http.ts
Normal file
|
|
@ -0,0 +1,519 @@
|
|||
/**
|
||||
* Shared HTTP plumbing for provider adapters: a fetch wrapper with timeouts,
|
||||
* an SSE parser over Web Streams, and one place that turns provider failures
|
||||
* into `AIError`s. Uses only global `fetch` — no SDKs.
|
||||
*
|
||||
* Security rule for this whole folder: an API key must never reach an error
|
||||
* message, a log line or a thrown value. Everything user-visible that contains
|
||||
* provider text goes through `scrubSecrets` first.
|
||||
*/
|
||||
import { AIError, type AIErrorCode } from '../types.ts';
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Secret scrubbing */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const KEY_PATTERNS: RegExp[] = [
|
||||
/\bsk-[A-Za-z0-9_-]{8,}/g, // OpenAI / Anthropic / OpenRouter style
|
||||
/\bAIza[0-9A-Za-z_-]{16,}/g, // Google API keys
|
||||
/\bgsk_[A-Za-z0-9]{8,}/g, // Groq
|
||||
/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
|
||||
/([?&](?:key|api_key|apikey|access_token)=)[^&\s"']+/gi,
|
||||
];
|
||||
|
||||
/** Removes known secrets (and anything that looks like an API key) from provider text. */
|
||||
export function scrubSecrets(text: string, secrets: Array<string | null | undefined> = []): string {
|
||||
let out = text;
|
||||
for (const s of secrets) {
|
||||
if (s && s.length >= 4) out = out.split(s).join('[redacted]');
|
||||
}
|
||||
for (const re of KEY_PATTERNS) {
|
||||
out = out.replace(re, (_m, prefix?: unknown) => (typeof prefix === 'string' ? `${prefix}[redacted]` : '[redacted]'));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function clip(text: string, max = 800): string {
|
||||
const t = text.trim();
|
||||
return t.length > max ? `${t.slice(0, max)}…` : t;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Error mapping */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface ErrorContext {
|
||||
/** Human provider label used in messages ("OpenAI", "Ollama"). */
|
||||
provider: string;
|
||||
/** Values that must never appear in an error (API key, custom header values). */
|
||||
secrets?: Array<string | null | undefined>;
|
||||
}
|
||||
|
||||
interface ProviderErrorFields {
|
||||
message: string;
|
||||
type: string;
|
||||
code: string;
|
||||
/** Whole body as text, used for keyword matching (quota ids, reasons, ...). */
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/** Pulls message/type/code out of the error envelopes used by the providers we talk to. */
|
||||
export function extractProviderError(body: unknown): ProviderErrorFields {
|
||||
let node: unknown = body;
|
||||
if (typeof node === 'string') {
|
||||
const text = node;
|
||||
try {
|
||||
node = JSON.parse(text);
|
||||
} catch {
|
||||
return { message: text.trim().startsWith('<') ? '' : text.trim(), type: '', code: '', raw: text };
|
||||
}
|
||||
}
|
||||
const raw = safeStringify(node);
|
||||
// Gemini sometimes wraps the error object in an array.
|
||||
if (Array.isArray(node)) node = node[0];
|
||||
if (!isRecord(node)) return { message: '', type: '', code: '', raw };
|
||||
const err = node.error ?? node;
|
||||
// Ollama: {"error": "model 'x' not found"}
|
||||
if (typeof err === 'string') return { message: err, type: '', code: '', raw };
|
||||
if (!isRecord(err)) return { message: '', type: '', code: '', raw };
|
||||
const message = typeof err.message === 'string' ? err.message : typeof node.message === 'string' ? node.message : '';
|
||||
// OpenAI: error.type + error.code; Anthropic: error.type; Gemini: error.status (+ numeric code).
|
||||
const type = [err.type, err.status].filter((v): v is string => typeof v === 'string').join(' ');
|
||||
const code = typeof err.code === 'string' ? err.code : '';
|
||||
return { message, type, code, raw };
|
||||
}
|
||||
|
||||
const CONTEXT_RE =
|
||||
/context[_ ]length|context[_ ]window|maximum context|max(?:imum)? (?:input )?tokens? (?:limit|allowed)|too many (?:input )?tokens|prompt is too long|input is too long|input token count|exceeds? the (?:model'?s? )?(?:maximum|max)[^.]{0,40}tokens|reduce the length|request[_ ]too[_ ]large|exceed_context_size|n_ctx/i;
|
||||
const QUOTA_RE =
|
||||
/insufficient[_ ]quota|billing|credit balance|exceeded your current quota|out of credits|insufficient (?:credits|funds|balance)|payment required|spend(?:ing)? limit/i;
|
||||
const AUTH_RE = /invalid[_ ]api[_ ]key|incorrect api key|api key not valid|api_key_invalid|authentication[_ ]error|invalid x-api-key|unauthorized|no auth credentials/i;
|
||||
const FILTER_RE = /content[_ ]filter|content[_ ]policy|content management policy|safety system|responsible ai/i;
|
||||
|
||||
function classify(status: number | undefined, f: ProviderErrorFields): AIErrorCode {
|
||||
const hay = `${f.type} ${f.code} ${f.message}`;
|
||||
const s = status ?? 0;
|
||||
|
||||
// Provider error names win over the status code: they are also all we have
|
||||
// for errors delivered inside an already-open (HTTP 200) stream.
|
||||
if (/authentication_error|invalid_api_key|permission_error|PERMISSION_DENIED|UNAUTHENTICATED/.test(hay)) return 'auth';
|
||||
if (/insufficient_quota|billing_error/.test(hay)) return 'quota';
|
||||
if (/context_length_exceeded|request_too_large/.test(hay)) return 'context_length';
|
||||
if (/overloaded_error|server_error|api_error|UNAVAILABLE|INTERNAL\b/.test(`${f.type} ${f.code}`)) return 'server';
|
||||
|
||||
if (s === 401 || s === 403) return 'auth';
|
||||
if (s === 402) return 'quota';
|
||||
if (s === 429 || /rate_limit|RESOURCE_EXHAUSTED/.test(hay)) {
|
||||
// Gemini words every 429 as "exceeded your current quota"; its details say
|
||||
// whether it is a per-minute limit (retry soon) or a daily/billing cap.
|
||||
if (/PerMinute|retryDelay|RetryInfo/.test(f.raw) && !/PerDay/.test(f.raw)) return 'rate_limit';
|
||||
return QUOTA_RE.test(hay) || /PerDay/.test(f.raw) ? 'quota' : 'rate_limit';
|
||||
}
|
||||
if (s === 404 || /not_found_error|model_not_found|NOT_FOUND/.test(hay)) return 'not_found';
|
||||
if (s === 408 || s === 504 || /DEADLINE_EXCEEDED/.test(hay)) return 'timeout';
|
||||
if (s === 413) return 'context_length';
|
||||
if (s >= 500) return 'server';
|
||||
// Remaining 4xx (and status-less stream errors).
|
||||
if (AUTH_RE.test(hay)) return 'auth'; // Gemini answers a bad key with 400 INVALID_ARGUMENT
|
||||
if (CONTEXT_RE.test(hay)) return 'context_length';
|
||||
if (QUOTA_RE.test(hay)) return 'quota'; // Anthropic: 400 "credit balance is too low"
|
||||
if (FILTER_RE.test(hay)) return 'content_filter';
|
||||
if (s === 0 && !f.message) return 'server';
|
||||
return 'bad_request';
|
||||
}
|
||||
|
||||
function friendlyMessage(code: AIErrorCode, provider: string): string {
|
||||
switch (code) {
|
||||
case 'auth':
|
||||
return `${provider} rejected the API key.`;
|
||||
case 'rate_limit':
|
||||
return `${provider} is rate limiting requests. Try again in a moment.`;
|
||||
case 'quota':
|
||||
return `${provider} reports that the account is out of quota or credit.`;
|
||||
case 'context_length':
|
||||
return `The conversation is too long for this ${provider} model.`;
|
||||
case 'not_found':
|
||||
return `${provider} could not find that model or endpoint.`;
|
||||
case 'content_filter':
|
||||
return `${provider} blocked the request with its content filter.`;
|
||||
case 'server':
|
||||
return `${provider} is having trouble right now. Try again shortly.`;
|
||||
case 'timeout':
|
||||
return `${provider} took too long to respond.`;
|
||||
case 'bad_request':
|
||||
return `${provider} rejected the request.`;
|
||||
default:
|
||||
return `${provider} returned an error.`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The single place that maps "HTTP status + provider error body" to an AIError.
|
||||
* Pass `status: undefined` for errors that arrive as events inside a stream.
|
||||
*/
|
||||
export function errorFromProvider(ctx: ErrorContext, status: number | undefined, body: unknown): AIError {
|
||||
const fields = extractProviderError(body);
|
||||
const code = classify(status, fields);
|
||||
const detail = fields.message || (status ? `HTTP ${status}` : '');
|
||||
return new AIError(code, friendlyMessage(code, ctx.provider), {
|
||||
status,
|
||||
providerMessage: detail ? clip(scrubSecrets(detail, ctx.secrets)) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
/** Maps anything thrown by `fetch` / a body read to an AIError. */
|
||||
export function errorFromThrown(ctx: ErrorContext, err: unknown, abortReason?: 'timeout' | 'aborted' | null): AIError {
|
||||
if (err instanceof AIError) return err;
|
||||
const name = isRecord(err) && typeof err.name === 'string' ? err.name : '';
|
||||
if (abortReason === 'timeout' || name === 'TimeoutError') {
|
||||
return new AIError('timeout', friendlyMessage('timeout', ctx.provider), { cause: err });
|
||||
}
|
||||
if (abortReason === 'aborted' || name === 'AbortError') {
|
||||
return new AIError('aborted', 'The request was cancelled.', { cause: err });
|
||||
}
|
||||
// undici reports DNS/connection failures as TypeError("fetch failed") with the real reason in `cause`.
|
||||
const cause = isRecord(err) && isRecord(err.cause) ? err.cause : undefined;
|
||||
const parts = [
|
||||
cause && typeof cause.code === 'string' ? cause.code : '',
|
||||
cause && typeof cause.message === 'string' ? cause.message : '',
|
||||
err instanceof Error ? err.message : String(err),
|
||||
].filter(Boolean);
|
||||
return new AIError('network', `Could not reach ${ctx.provider}. Check the address and your connection.`, {
|
||||
providerMessage: clip(scrubSecrets(parts.join(': '), ctx.secrets)),
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* fetch with timeouts */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface HttpOptions extends ErrorContext {
|
||||
method?: 'GET' | 'POST';
|
||||
headers?: Record<string, string>;
|
||||
/** Serialised as JSON when present. */
|
||||
body?: unknown;
|
||||
signal?: AbortSignal;
|
||||
/** Time allowed until response headers arrive (default 60s). */
|
||||
firstByteTimeoutMs?: number;
|
||||
/** Hard cap on the whole exchange (default 30 min — streams are long). */
|
||||
totalTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface HttpResponse {
|
||||
response: Response;
|
||||
ctx: ErrorContext;
|
||||
/** Abort the exchange from our side (used by the idle watchdog). */
|
||||
abort(reason: 'timeout' | 'aborted'): void;
|
||||
abortReason(): 'timeout' | 'aborted' | null;
|
||||
/** Clears timers/listeners. Safe to call more than once. */
|
||||
cleanup(): void;
|
||||
}
|
||||
|
||||
export const DEFAULT_FIRST_BYTE_TIMEOUT_MS = 60_000;
|
||||
export const DEFAULT_TOTAL_TIMEOUT_MS = 30 * 60_000;
|
||||
/** Longest silence tolerated inside an open stream before we give up. */
|
||||
export const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000;
|
||||
|
||||
/**
|
||||
* Sends a request and resolves once headers are in. Does NOT check `response.ok`
|
||||
* (callers decide; see `ensureOk`). Throws AIError for abort/timeout/network.
|
||||
*/
|
||||
export async function httpRequest(url: string, opts: HttpOptions): Promise<HttpResponse> {
|
||||
const ctx: ErrorContext = { provider: opts.provider, secrets: opts.secrets };
|
||||
if (opts.signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
|
||||
|
||||
const controller = new AbortController();
|
||||
let reason: 'timeout' | 'aborted' | null = null;
|
||||
const abort = (r: 'timeout' | 'aborted') => {
|
||||
if (reason) return;
|
||||
reason = r;
|
||||
controller.abort();
|
||||
};
|
||||
const onUserAbort = () => abort('aborted');
|
||||
opts.signal?.addEventListener('abort', onUserAbort, { once: true });
|
||||
|
||||
const firstByte = setTimeout(() => abort('timeout'), opts.firstByteTimeoutMs ?? DEFAULT_FIRST_BYTE_TIMEOUT_MS);
|
||||
const total = setTimeout(() => abort('timeout'), opts.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS);
|
||||
let cleaned = false;
|
||||
const cleanup = () => {
|
||||
if (cleaned) return;
|
||||
cleaned = true;
|
||||
clearTimeout(firstByte);
|
||||
clearTimeout(total);
|
||||
opts.signal?.removeEventListener('abort', onUserAbort);
|
||||
};
|
||||
|
||||
const headers: Record<string, string> = { ...opts.headers };
|
||||
let body: string | undefined;
|
||||
if (opts.body !== undefined) {
|
||||
body = JSON.stringify(opts.body);
|
||||
if (!hasHeader(headers, 'content-type')) headers['Content-Type'] = 'application/json';
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: opts.method ?? (body === undefined ? 'GET' : 'POST'),
|
||||
headers,
|
||||
body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(firstByte);
|
||||
return { response, ctx, abort, abortReason: () => reason, cleanup };
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
throw errorFromThrown(ctx, err, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the body of a failed response and throws the mapped AIError. Returns normally when `ok`. */
|
||||
export async function ensureOk(http: HttpResponse): Promise<void> {
|
||||
if (http.response.ok) return;
|
||||
let text = '';
|
||||
try {
|
||||
text = await http.response.text();
|
||||
} catch {
|
||||
/* body unreadable — the status code alone still maps to something useful */
|
||||
} finally {
|
||||
http.cleanup();
|
||||
}
|
||||
throw errorFromProvider(http.ctx, http.response.status, text);
|
||||
}
|
||||
|
||||
/** Request + status check + JSON parse, for non-streaming calls such as model discovery. */
|
||||
export async function requestJson<T = unknown>(url: string, opts: HttpOptions): Promise<T> {
|
||||
const http = await httpRequest(url, opts);
|
||||
await ensureOk(http);
|
||||
try {
|
||||
const text = await http.response.text();
|
||||
try {
|
||||
return JSON.parse(text) as T;
|
||||
} catch {
|
||||
throw new AIError('invalid_response', `${opts.provider} sent a response that was not valid JSON.`, {
|
||||
providerMessage: clip(scrubSecrets(text, opts.secrets), 300),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
||||
} finally {
|
||||
http.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Server-sent events */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface SSEMessage {
|
||||
/** Value of the `event:` field, when the server sent one. */
|
||||
event?: string;
|
||||
/** All `data:` lines of the event joined with "\n". */
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface SSEOptions {
|
||||
/** Abort when no bytes arrive for this long. */
|
||||
idleTimeoutMs?: number;
|
||||
onIdle?: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a text/event-stream body. Handles `\n`, `\r\n` and bare `\r` line
|
||||
* endings (also when split across chunks), multi-line `data:`, comment lines,
|
||||
* a UTF-8 BOM, and a final event that is not followed by a blank line.
|
||||
* Iteration ends at end-of-stream or at the OpenAI-style `data: [DONE]` sentinel.
|
||||
*/
|
||||
export async function* parseSSE(stream: ReadableStream<Uint8Array>, opts: SSEOptions = {}): AsyncGenerator<SSEMessage> {
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
let buffer = '';
|
||||
let dataLines: string[] = [];
|
||||
let eventName: string | undefined;
|
||||
let sawCR = false; // previous chunk ended in "\r": swallow a leading "\n"
|
||||
let first = true;
|
||||
|
||||
const takeLine = (line: string): SSEMessage | 'done' | null => {
|
||||
if (line === '') {
|
||||
// Blank line dispatches the event.
|
||||
if (dataLines.length === 0) {
|
||||
eventName = undefined;
|
||||
return null;
|
||||
}
|
||||
const msg: SSEMessage = { data: dataLines.join('\n') };
|
||||
if (eventName !== undefined) msg.event = eventName;
|
||||
dataLines = [];
|
||||
eventName = undefined;
|
||||
return msg.data === '[DONE]' ? 'done' : msg;
|
||||
}
|
||||
if (line.startsWith(':')) return null; // comment / keep-alive
|
||||
const colon = line.indexOf(':');
|
||||
const field = colon === -1 ? line : line.slice(0, colon);
|
||||
let value = colon === -1 ? '' : line.slice(colon + 1);
|
||||
if (value.startsWith(' ')) value = value.slice(1);
|
||||
if (field === 'data') dataLines.push(value);
|
||||
else if (field === 'event') eventName = value;
|
||||
// `id` and `retry` are irrelevant for one-shot completions.
|
||||
return null;
|
||||
};
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await readWithIdle(reader, opts);
|
||||
if (done) break;
|
||||
let chunk = decoder.decode(value, { stream: true });
|
||||
if (first && chunk.length > 0) {
|
||||
first = false;
|
||||
if (chunk.charCodeAt(0) === 0xfeff) chunk = chunk.slice(1);
|
||||
}
|
||||
if (sawCR && chunk.startsWith('\n')) chunk = chunk.slice(1);
|
||||
if (chunk.length > 0) sawCR = chunk.endsWith('\r');
|
||||
buffer += chunk;
|
||||
|
||||
let start = 0;
|
||||
for (;;) {
|
||||
const idx = indexOfLineEnd(buffer, start);
|
||||
if (idx === -1) break;
|
||||
const line = buffer.slice(start, idx);
|
||||
start = buffer[idx] === '\r' && buffer[idx + 1] === '\n' ? idx + 2 : idx + 1;
|
||||
const out = takeLine(line);
|
||||
if (out === 'done') return;
|
||||
if (out) yield out;
|
||||
}
|
||||
buffer = buffer.slice(start);
|
||||
}
|
||||
// Flush: tolerate a last line / last event without the terminating newline(s).
|
||||
buffer += decoder.decode();
|
||||
if (buffer.length > 0) {
|
||||
const out = takeLine(buffer);
|
||||
if (out === 'done') return;
|
||||
if (out) yield out;
|
||||
}
|
||||
const tail = takeLine('');
|
||||
if (tail && tail !== 'done') yield tail;
|
||||
} finally {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already closed or errored */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function indexOfLineEnd(s: string, from: number): number {
|
||||
for (let i = from; i < s.length; i++) {
|
||||
const c = s.charCodeAt(i);
|
||||
if (c === 10 || c === 13) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
async function readWithIdle(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
opts: SSEOptions,
|
||||
): Promise<Awaited<ReturnType<ReadableStreamDefaultReader<Uint8Array>['read']>>> {
|
||||
if (!opts.idleTimeoutMs) return reader.read();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const idle = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
opts.onIdle?.();
|
||||
const err = new Error('stream idle timeout');
|
||||
err.name = 'TimeoutError';
|
||||
reject(err);
|
||||
}, opts.idleTimeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([reader.read(), idle]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterates the SSE events of an open response, translating read failures
|
||||
* (dropped connection, abort, idle timeout) into AIErrors and always releasing
|
||||
* the request's timers. `data` is JSON-parsed; unparseable events are skipped.
|
||||
*/
|
||||
export async function* sseJson(http: HttpResponse, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS): AsyncGenerator<{ event?: string; data: unknown }> {
|
||||
const body = http.response.body;
|
||||
if (!body) {
|
||||
http.cleanup();
|
||||
throw new AIError('invalid_response', `${http.ctx.provider} returned an empty response.`);
|
||||
}
|
||||
const events = parseSSE(body, { idleTimeoutMs, onIdle: () => http.abort('timeout') });
|
||||
try {
|
||||
for (;;) {
|
||||
let next: IteratorResult<SSEMessage>;
|
||||
try {
|
||||
next = await events.next();
|
||||
} catch (err) {
|
||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
||||
}
|
||||
if (next.done) return;
|
||||
let data: unknown;
|
||||
try {
|
||||
data = JSON.parse(next.value.data);
|
||||
} catch {
|
||||
continue; // keep-alives such as OpenRouter's ": OPENROUTER PROCESSING" variants, or junk
|
||||
}
|
||||
yield next.value.event !== undefined ? { event: next.value.event, data } : { data };
|
||||
}
|
||||
} finally {
|
||||
await events.return(undefined).catch(() => undefined);
|
||||
http.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Small shared utilities */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export function isRecord(v: unknown): v is Record<string, unknown> {
|
||||
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
||||
}
|
||||
|
||||
function hasHeader(headers: Record<string, string>, name: string): boolean {
|
||||
const n = name.toLowerCase();
|
||||
return Object.keys(headers).some((k) => k.toLowerCase() === n);
|
||||
}
|
||||
|
||||
function safeStringify(v: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(v) ?? '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/** Trims whitespace and trailing slashes from a base URL. */
|
||||
export function trimBaseUrl(url: string): string {
|
||||
return url.trim().replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/** Host part of a URL for user-facing messages; never includes credentials or query. */
|
||||
export function hostOf(url: string): string {
|
||||
try {
|
||||
return new URL(url).host;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
/** True for loopback / private-network hosts, where a local model server may need minutes to load a model. */
|
||||
export function isLocalUrl(url: string): boolean {
|
||||
try {
|
||||
const h = new URL(url).hostname.replace(/^\[|\]$/g, '');
|
||||
return (
|
||||
h === 'localhost' ||
|
||||
h === '::1' ||
|
||||
h.endsWith('.local') ||
|
||||
h.endsWith('.internal') ||
|
||||
/^127\./.test(h) ||
|
||||
/^10\./.test(h) ||
|
||||
/^192\.168\./.test(h) ||
|
||||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
|
||||
h === 'host.docker.internal'
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
26
server/ai/providers/index.ts
Normal file
26
server/ai/providers/index.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { AIError, type ProviderAdapter, type ProviderKind } from '../types.ts';
|
||||
import { anthropicAdapter } from './anthropic.ts';
|
||||
import { geminiAdapter } from './gemini.ts';
|
||||
import { ollamaAdapter } from './ollama.ts';
|
||||
import { openaiAdapter } from './openai.ts';
|
||||
import { openaiCompatibleAdapter } from './openaiCompatible.ts';
|
||||
|
||||
const ADAPTERS: Record<ProviderKind, ProviderAdapter> = {
|
||||
openai: openaiAdapter,
|
||||
anthropic: anthropicAdapter,
|
||||
gemini: geminiAdapter,
|
||||
openai_compatible: openaiCompatibleAdapter,
|
||||
ollama: ollamaAdapter,
|
||||
};
|
||||
|
||||
export function getAdapter(kind: ProviderKind): ProviderAdapter {
|
||||
const adapter = ADAPTERS[kind];
|
||||
if (!adapter) throw new AIError('unsupported', `Unknown provider type "${kind}".`);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
export function listAdapterMeta(): Array<{ kind: ProviderKind } & ProviderAdapter['meta']> {
|
||||
return (Object.keys(ADAPTERS) as ProviderKind[]).map((kind) => ({ kind, ...ADAPTERS[kind].meta }));
|
||||
}
|
||||
|
||||
export { CATALOG_UPDATED, lookupModelDefaults } from './catalog.ts';
|
||||
102
server/ai/providers/ollama.ts
Normal file
102
server/ai/providers/ollama.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
/**
|
||||
* Ollama (local models). Chat goes through Ollama's OpenAI-compatible endpoint
|
||||
* (`{base}/v1/chat/completions`), which handles tools and streaming; discovery
|
||||
* uses the native API because it reports what each model can do:
|
||||
* GET /api/tags -> installed models
|
||||
* POST /api/show -> { capabilities: ["completion","tools","vision","thinking"], model_info }
|
||||
* https://docs.ollama.com/api · https://docs.ollama.com/openai
|
||||
*/
|
||||
import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts';
|
||||
import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts';
|
||||
import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
||||
import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts';
|
||||
|
||||
const LABEL = 'Ollama';
|
||||
const DEFAULT_BASE_URL = 'http://localhost:11434';
|
||||
|
||||
function root(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/(v1|api)$/, '');
|
||||
}
|
||||
|
||||
function ctx(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
const h: Record<string, string> = { ...cfg.headers };
|
||||
if (cfg.apiKey) h.Authorization = `Bearer ${cfg.apiKey}`; // only needed behind an authenticating proxy
|
||||
return h;
|
||||
}
|
||||
|
||||
function compat(cfg: ProviderConfig): CompatOptions {
|
||||
return { kind: 'ollama', label: LABEL, baseUrl: `${root(cfg)}/v1`, apiKey: cfg.apiKey, headers: cfg.headers };
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
let tags: unknown;
|
||||
try {
|
||||
tags = await requestJson(`${root(cfg)}/api/tags`, { ...ctx(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs ?? 15_000, totalTimeoutMs: timeoutMs ?? 30_000 });
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'network') {
|
||||
throw new AIError('network', `Could not reach Ollama at ${root(cfg)}. Is it running, and reachable from the Study Desk container?`, { retryable: true });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const list = isRecord(tags) && Array.isArray(tags.models) ? tags.models.filter(isRecord) : [];
|
||||
const models: DiscoveredModel[] = [];
|
||||
// /api/show is one cheap local call per model; cap it so a huge library cannot stall the settings page.
|
||||
await Promise.all(
|
||||
list.slice(0, 60).map(async (m) => {
|
||||
const id = str(m.model) || str(m.name);
|
||||
if (!id) return;
|
||||
const model: DiscoveredModel = { id, label: str(m.name) || id, priceIn: 0, priceOut: 0 };
|
||||
try {
|
||||
const show = await requestJson<Record<string, unknown>>(`${root(cfg)}/api/show`, {
|
||||
...ctx(cfg),
|
||||
headers: headers(cfg),
|
||||
body: { model: id },
|
||||
signal,
|
||||
firstByteTimeoutMs: 10_000,
|
||||
totalTimeoutMs: 10_000,
|
||||
});
|
||||
if (Array.isArray(show.capabilities)) {
|
||||
const caps = show.capabilities.map(String);
|
||||
if (!caps.includes('completion')) return; // embedding-only model
|
||||
const c: Partial<ModelCapabilities> = { tools: caps.includes('tools'), vision: caps.includes('vision'), reasoning: caps.includes('thinking'), pdf: false, temperature: true };
|
||||
model.capabilities = c;
|
||||
}
|
||||
if (isRecord(show.model_info)) {
|
||||
const key = Object.keys(show.model_info).find((k) => k.endsWith('.context_length'));
|
||||
if (key && num(show.model_info[key])) model.contextWindow = num(show.model_info[key]);
|
||||
}
|
||||
} catch {
|
||||
/* older Ollama versions: keep the model with unknown capabilities */
|
||||
}
|
||||
models.push(model);
|
||||
}),
|
||||
);
|
||||
return models.sort((a, b) => a.id.localeCompare(b.id));
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => {
|
||||
const count = (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length;
|
||||
return count === 0 ? { modelCount: 0, note: 'Connected to Ollama, but no models are installed. Run `ollama pull <model>` first.' } : { modelCount: count };
|
||||
});
|
||||
}
|
||||
|
||||
export const ollamaAdapter: ProviderAdapter = {
|
||||
kind: 'ollama',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: false,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://docs.ollama.com',
|
||||
keyHint: 'Not needed',
|
||||
},
|
||||
stream: (cfg, req) => streamChatCompletions(compat(cfg), req),
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
313
server/ai/providers/openai.ts
Normal file
313
server/ai/providers/openai.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* OpenAI — Responses API (`POST /v1/responses`, SSE).
|
||||
* https://platform.openai.com/docs/api-reference/responses
|
||||
* https://platform.openai.com/docs/guides/function-calling
|
||||
* https://platform.openai.com/docs/guides/reasoning
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - Requests are stateless (`store: false`). Reasoning models need their reasoning
|
||||
* items passed back between tool calls, so encrypted reasoning is requested via
|
||||
* `include: ["reasoning.encrypted_content"]` and every turn's raw output items are
|
||||
* kept in `providerData` and replayed verbatim.
|
||||
* - Reasoning models reject `temperature`; classic models reject `reasoning`.
|
||||
* - The accepted `reasoning.effort` values differ by model generation. When a model
|
||||
* rejects the chosen value the request is retried once with a safe one.
|
||||
* - Tool-call arguments arrive as deltas keyed by output item.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'OpenAI';
|
||||
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
|
||||
|
||||
function base(cfg: ProviderConfig): string {
|
||||
const url = trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL);
|
||||
return /\/v\d+$/.test(url) ? url : `${url}/v1`;
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an OpenAI API key in Settings to use OpenAI models.');
|
||||
return { ...cfg.headers, Authorization: `Bearer ${cfg.apiKey}` };
|
||||
}
|
||||
|
||||
/** Whether a model is a reasoning model, when the caller did not say. */
|
||||
function isReasoningModel(req: ChatRequest): boolean {
|
||||
if (req.capabilities?.reasoning !== undefined) return req.capabilities.reasoning;
|
||||
const known = lookupModelDefaults('openai', req.model).capabilities?.reasoning;
|
||||
if (known !== undefined) return known;
|
||||
return /^(o\d|gpt-[5-9]|gpt-\d{2})/.test(req.model) && !/chat/.test(req.model);
|
||||
}
|
||||
|
||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'text' && p.text) out.push({ type: 'input_text', text: p.text });
|
||||
else if (p.type === 'image') {
|
||||
if (caps.vision === false) out.push({ type: 'input_text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else out.push({ type: 'input_image', image_url: dataUrl(p.mime, p.data) });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ type: 'input_file', filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) });
|
||||
else if (p.mime.startsWith('text/')) out.push({ type: 'input_text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else out.push({ type: 'input_text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function toInput(messages: ChatMessage[], caps: Partial<ModelCapabilities>, dropReasoning: boolean): Json[] {
|
||||
const input: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
if (msg.role === 'assistant') {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'openai' && isRecord(pd.data) && Array.isArray(pd.data.output)) {
|
||||
// Same provider: replay the output items exactly (reasoning items included).
|
||||
for (const item of pd.data.output.filter(isRecord)) {
|
||||
if (item.type === 'reasoning' && (dropReasoning || !item.encrypted_content)) continue;
|
||||
input.push(item);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
||||
if (text) input.push({ role: 'assistant', content: [{ type: 'output_text', text }] });
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_call') input.push({ type: 'function_call', call_id: p.id, name: p.name, arguments: JSON.stringify(p.input ?? {}) });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_result') input.push({ type: 'function_call_output', call_id: p.toolCallId, output: p.output || '(no output)' });
|
||||
}
|
||||
const content = userContent(msg.content.filter((p) => p.type !== 'tool_result'), caps);
|
||||
if (content.length > 0) input.push({ role: 'user', content });
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
interface Flags {
|
||||
/** Value sent as reasoning.effort, or null to omit the field. */
|
||||
effort: string | null;
|
||||
summary: boolean;
|
||||
temperature: boolean;
|
||||
dropReasoning: boolean;
|
||||
include: boolean;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, flags: Flags): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { model: req.model, input: toInput(req.messages, caps, flags.dropReasoning), stream: true, store: false };
|
||||
if (req.system?.trim()) body.instructions = req.system.trim();
|
||||
if (req.maxOutputTokens) body.max_output_tokens = Math.max(16, req.maxOutputTokens);
|
||||
if (caps.tools !== false && req.tools?.length) {
|
||||
body.tools = req.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters }, strict: false }));
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', name: choice.name };
|
||||
}
|
||||
if (isReasoningModel(req)) {
|
||||
const reasoning: Json = {};
|
||||
if (flags.effort) reasoning.effort = flags.effort;
|
||||
if (flags.summary) reasoning.summary = 'auto';
|
||||
if (Object.keys(reasoning).length > 0) body.reasoning = reasoning;
|
||||
if (flags.include) body.include = ['reasoning.encrypted_content'];
|
||||
} else if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) {
|
||||
body.temperature = req.temperature;
|
||||
}
|
||||
if (req.jsonMode) body.text = { format: { type: 'json_object' } };
|
||||
return body;
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const flags: Flags = {
|
||||
// "none" is only understood by recent models; if it is rejected we fall back to "low".
|
||||
effort: req.reasoningEffort ?? null,
|
||||
summary: true,
|
||||
temperature: true,
|
||||
dropReasoning: false,
|
||||
include: true,
|
||||
};
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(`${base(cfg)}/responses`, { ...ctx, headers: headers(cfg), body: buildBody(req, flags), signal: req.signal });
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 5) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (flags.effort && /reasoning\.effort|reasoning_effort|effort/.test(detail)) flags.effort = flags.effort === 'none' ? 'minimal' : flags.effort === 'minimal' ? 'low' : null;
|
||||
else if (flags.summary && /summary/.test(detail)) flags.summary = false;
|
||||
else if (flags.include && /include|encrypted_content/.test(detail)) flags.include = false;
|
||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
||||
else if (!flags.dropReasoning && /reasoning/.test(detail) && /item|input/.test(detail)) flags.dropReasoning = true;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const args = new Map<string, string>();
|
||||
let final: Json | null = null;
|
||||
let reasoningEmitted = false;
|
||||
let streamedText = '';
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
switch (data.type) {
|
||||
case 'response.output_text.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) {
|
||||
streamedText += delta;
|
||||
yield { type: 'text', delta };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'response.refusal.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) yield { type: 'text', delta };
|
||||
break;
|
||||
}
|
||||
case 'response.reasoning_summary_part.added':
|
||||
if (reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
||||
break;
|
||||
case 'response.reasoning_summary_text.delta':
|
||||
case 'response.reasoning_text.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) {
|
||||
reasoningEmitted = true;
|
||||
yield { type: 'reasoning', delta };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'response.function_call_arguments.delta':
|
||||
args.set(str(data.item_id), (args.get(str(data.item_id)) ?? '') + str(data.delta));
|
||||
break;
|
||||
case 'response.completed':
|
||||
case 'response.incomplete':
|
||||
if (isRecord(data.response)) final = data.response;
|
||||
break;
|
||||
case 'response.failed':
|
||||
throw errorFromProvider(ctx, undefined, isRecord(data.response) && isRecord(data.response.error) ? { error: data.response.error } : data);
|
||||
case 'error':
|
||||
throw errorFromProvider(ctx, undefined, isRecord(data.error) ? data : { error: data });
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!final) throw new AIError('network', 'The connection to OpenAI closed before the response finished.', { retryable: true });
|
||||
|
||||
const incomplete = isRecord(final.incomplete_details) ? str(final.incomplete_details.reason) : '';
|
||||
const truncated = incomplete === 'max_output_tokens';
|
||||
const output = Array.isArray(final.output) ? final.output.filter(isRecord) : [];
|
||||
const content: ContentPart[] = [];
|
||||
const replay: Json[] = [];
|
||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
let refused = false;
|
||||
let text = '';
|
||||
|
||||
for (const item of output) {
|
||||
if (item.type === 'message') {
|
||||
for (const part of Array.isArray(item.content) ? item.content.filter(isRecord) : []) {
|
||||
if (part.type === 'output_text') text += str(part.text);
|
||||
else if (part.type === 'refusal') {
|
||||
refused = true;
|
||||
text += str(part.refusal);
|
||||
}
|
||||
}
|
||||
replay.push(item);
|
||||
} else if (item.type === 'function_call') {
|
||||
const name = str(item.name);
|
||||
const raw = str(item.arguments) || args.get(str(item.id)) || '';
|
||||
let input = parseArgsStrict(raw);
|
||||
if (!input) {
|
||||
if (truncated) continue;
|
||||
input = parseToolArguments(raw, LABEL, name);
|
||||
}
|
||||
calls.push({ id: str(item.call_id) || str(item.id), name, input });
|
||||
replay.push(item);
|
||||
} else if (item.type === 'reasoning') {
|
||||
replay.push(item);
|
||||
}
|
||||
}
|
||||
if (!text && streamedText) text = streamedText;
|
||||
if (text) content.push({ type: 'text', text });
|
||||
for (const call of calls) {
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
yield { type: 'tool_call', call };
|
||||
}
|
||||
|
||||
let stopReason: StopReason = 'end';
|
||||
if (calls.length > 0) stopReason = 'tool_use';
|
||||
else if (refused || incomplete === 'content_filter') stopReason = 'refusal';
|
||||
else if (truncated) stopReason = 'max_tokens';
|
||||
else if (incomplete) stopReason = 'other';
|
||||
|
||||
const u = isRecord(final.usage) ? final.usage : {};
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
if (replay.length > 0) message.providerData = { kind: 'openai', data: { output: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason,
|
||||
usage: makeUsage({
|
||||
input: num(u.input_tokens),
|
||||
output: num(u.output_tokens),
|
||||
cacheRead: isRecord(u.input_tokens_details) ? num(u.input_tokens_details.cached_tokens) : 0,
|
||||
reasoning: isRecord(u.output_tokens_details) ? num(u.output_tokens_details.reasoning_tokens) : 0,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const res = await requestJson<Json>(`${base(cfg)}/models`, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
||||
const found: DiscoveredModel[] = [];
|
||||
for (const m of data) {
|
||||
const id = str(m.id);
|
||||
// Fine-tunes ("ft:...") are chat models too; search/codex-only variants are not useful here.
|
||||
if (!id || !isChatModelId(id) || /search|^codex-|-codex-max/.test(id)) continue;
|
||||
if (!/^(gpt|o\d|chat|ft:)/.test(id)) continue;
|
||||
found.push({ id });
|
||||
}
|
||||
found.sort((a, b) => b.id.localeCompare(a.id, undefined, { numeric: true }));
|
||||
return withCatalogDefaults('openai', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const openaiAdapter: ProviderAdapter = {
|
||||
kind: 'openai',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://platform.openai.com/api-keys',
|
||||
keyHint: 'sk-…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
363
server/ai/providers/openaiCompatible.ts
Normal file
363
server/ai/providers/openaiCompatible.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
/**
|
||||
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
|
||||
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
|
||||
* Ollama adapter.
|
||||
*
|
||||
* These servers agree on the basics and disagree on everything else, so this
|
||||
* adapter is deliberately forgiving:
|
||||
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
|
||||
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
|
||||
* once when a server answers 400 about them;
|
||||
* - usage may be missing, and may arrive on a chunk with no choices;
|
||||
* - tool-call argument fragments may arrive in any chunking, with or without ids;
|
||||
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
|
||||
* - the last chunk may lack a `finish_reason`.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type ProviderKind,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
|
||||
export function normalizeCompatBase(url: string): string {
|
||||
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
|
||||
try {
|
||||
const u = new URL(base);
|
||||
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
|
||||
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
|
||||
} catch {
|
||||
/* validated by the caller */
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
export interface CompatOptions {
|
||||
kind: ProviderKind;
|
||||
label: string;
|
||||
baseUrl: string;
|
||||
apiKey: string | null;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
|
||||
const blocks: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
||||
else if (p.type === 'image') {
|
||||
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
// Plain strings are understood by every server; arrays only by multimodal ones.
|
||||
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
|
||||
return blocks;
|
||||
}
|
||||
|
||||
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
|
||||
const caps = req.capabilities ?? {};
|
||||
const out: Json[] = [];
|
||||
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
|
||||
for (const msg of req.messages) {
|
||||
if (msg.role === 'assistant') {
|
||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
||||
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
|
||||
if (!text && calls.length === 0) continue;
|
||||
const wire: Json = { role: 'assistant', content: text || null };
|
||||
if (calls.length > 0) {
|
||||
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
|
||||
}
|
||||
out.push(wire);
|
||||
continue;
|
||||
}
|
||||
// Tool results must directly follow the assistant turn that requested them.
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
|
||||
}
|
||||
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
|
||||
if (rest.length > 0) {
|
||||
const content = userContent(rest, caps);
|
||||
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface BodyFlags {
|
||||
tools: boolean;
|
||||
usage: boolean;
|
||||
jsonMode: boolean;
|
||||
temperature: boolean;
|
||||
reasoning: boolean;
|
||||
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
|
||||
if (flags.usage) body.stream_options = { include_usage: true };
|
||||
if (flags.tools && caps.tools !== false && req.tools?.length) {
|
||||
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
|
||||
}
|
||||
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
|
||||
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
|
||||
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
|
||||
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
|
||||
return body;
|
||||
}
|
||||
|
||||
function mapFinish(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'stop':
|
||||
case 'eos':
|
||||
case 'end_turn':
|
||||
return 'end';
|
||||
case 'tool_calls':
|
||||
case 'function_call':
|
||||
return 'tool_use';
|
||||
case 'length':
|
||||
case 'max_tokens':
|
||||
return 'max_tokens';
|
||||
case 'content_filter':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other';
|
||||
}
|
||||
}
|
||||
|
||||
function authHeaders(o: CompatOptions): Record<string, string> {
|
||||
const h: Record<string, string> = { ...o.headers };
|
||||
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
function errorContext(o: CompatOptions): ErrorContext {
|
||||
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
|
||||
}
|
||||
|
||||
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(o);
|
||||
const base = normalizeCompatBase(o.baseUrl);
|
||||
const local = isLocalUrl(base);
|
||||
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(`${base}/chat/completions`, {
|
||||
...ctx,
|
||||
headers: authHeaders(o),
|
||||
body: buildBody(req, flags),
|
||||
signal: req.signal,
|
||||
// A local server may spend minutes loading a model into memory before the first byte.
|
||||
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
|
||||
});
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
|
||||
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
|
||||
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
|
||||
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
|
||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
||||
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
|
||||
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
|
||||
// The model (or server) cannot do tool calling at all.
|
||||
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
|
||||
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
|
||||
status: err.status,
|
||||
providerMessage: err.providerMessage,
|
||||
});
|
||||
}
|
||||
flags.tools = false;
|
||||
} else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
let text = '';
|
||||
let finish = '';
|
||||
let reasoningSeen = false;
|
||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
||||
const calls: { id: string; name: string; args: string }[] = [];
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
||||
if (isRecord(data.usage)) {
|
||||
const u = data.usage;
|
||||
usage.input = num(u.prompt_tokens) || usage.input;
|
||||
usage.output = num(u.completion_tokens) || usage.output;
|
||||
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
|
||||
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
|
||||
}
|
||||
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
|
||||
if (!isRecord(choice)) continue;
|
||||
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
|
||||
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
|
||||
if (!delta) continue;
|
||||
|
||||
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
|
||||
if (reasoning) {
|
||||
reasoningSeen = true;
|
||||
yield { type: 'reasoning', delta: reasoning };
|
||||
}
|
||||
const piece = typeof delta.content === 'string' ? delta.content : '';
|
||||
if (piece) {
|
||||
text += piece;
|
||||
yield { type: 'text', delta: piece };
|
||||
}
|
||||
if (Array.isArray(delta.tool_calls)) {
|
||||
for (const raw of delta.tool_calls) {
|
||||
if (!isRecord(raw)) continue;
|
||||
const fn = isRecord(raw.function) ? raw.function : {};
|
||||
// `index` identifies the call; servers that omit it send one whole call per entry.
|
||||
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
|
||||
let slot = index >= 0 ? calls[index] : undefined;
|
||||
if (!slot) {
|
||||
slot = { id: '', name: '', args: '' };
|
||||
if (index >= 0) calls[index] = slot;
|
||||
else calls.push(slot);
|
||||
}
|
||||
if (str(raw.id)) slot.id = str(raw.id);
|
||||
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
|
||||
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
|
||||
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const stop = finish ? mapFinish(finish) : 'end';
|
||||
const content: ContentPart[] = [];
|
||||
if (text) content.push({ type: 'text', text });
|
||||
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
for (const c of calls) {
|
||||
if (!c || !c.name) continue;
|
||||
let input = parseArgsStrict(c.args);
|
||||
if (!input) {
|
||||
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
|
||||
input = parseToolArguments(c.args, o.label, c.name);
|
||||
}
|
||||
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
|
||||
}
|
||||
for (const call of done) {
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
yield { type: 'tool_call', call };
|
||||
}
|
||||
if (content.length === 0 && !reasoningSeen && !finish) {
|
||||
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
|
||||
}
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
|
||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const base = normalizeCompatBase(o.baseUrl);
|
||||
let res: unknown;
|
||||
try {
|
||||
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
|
||||
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
|
||||
const found: DiscoveredModel[] = [];
|
||||
for (const m of list) {
|
||||
if (!isRecord(m)) continue;
|
||||
const id = str(m.id) || str(m.name);
|
||||
if (!id || !isChatModelId(id)) continue;
|
||||
const model: DiscoveredModel = { id };
|
||||
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
|
||||
// OpenRouter-style metadata, when present.
|
||||
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
|
||||
if (ctxLen) model.contextWindow = ctxLen;
|
||||
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
|
||||
if (isRecord(m.pricing)) {
|
||||
const pin = Number(m.pricing.prompt);
|
||||
const pout = Number(m.pricing.completion);
|
||||
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
|
||||
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
|
||||
}
|
||||
const caps: Partial<ModelCapabilities> = {};
|
||||
if (Array.isArray(m.supported_parameters)) {
|
||||
caps.tools = m.supported_parameters.includes('tools');
|
||||
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
|
||||
}
|
||||
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
|
||||
caps.vision = m.architecture.input_modalities.includes('image');
|
||||
}
|
||||
if (Object.keys(caps).length > 0) model.capabilities = caps;
|
||||
found.push(model);
|
||||
}
|
||||
return withCatalogDefaults(o.kind, found);
|
||||
}
|
||||
|
||||
function options(cfg: ProviderConfig): CompatOptions {
|
||||
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
|
||||
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
|
||||
return runTest(cfg.name || 'the server', async () => {
|
||||
const o = options(cfg);
|
||||
try {
|
||||
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
|
||||
// No model listing: prove the connection with a one-token completion instead.
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
|
||||
try {
|
||||
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
|
||||
if (ev.type === 'done') break;
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export const openaiCompatibleAdapter: ProviderAdapter = {
|
||||
kind: 'openai_compatible',
|
||||
meta: {
|
||||
label: 'OpenAI-compatible',
|
||||
defaultBaseUrl: null,
|
||||
requiresApiKey: false,
|
||||
requiresBaseUrl: true,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://openrouter.ai/docs/quickstart',
|
||||
keyHint: 'Optional for local servers',
|
||||
},
|
||||
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
|
||||
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
|
||||
test,
|
||||
};
|
||||
932
server/ai/providers/providers.test.ts
Normal file
932
server/ai/providers/providers.test.ts
Normal file
|
|
@ -0,0 +1,932 @@
|
|||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/**
|
||||
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
|
||||
* Wire shapes below are derived from what each adapter's parser reads.
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
|
||||
import { anthropicAdapter } from './anthropic.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
|
||||
import { parseSSE } from './http.ts';
|
||||
import { ollamaAdapter } from './ollama.ts';
|
||||
import { openaiAdapter } from './openai.ts';
|
||||
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream({
|
||||
start(c) {
|
||||
for (const ch of chunks) c.enqueue(ch);
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
|
||||
|
||||
/** One SSE event. */
|
||||
function ev(data: unknown, event?: string): string {
|
||||
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
|
||||
}
|
||||
const DONE = 'data: [DONE]\n\n';
|
||||
|
||||
function rechunk(s: string, n: number): string[] {
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** SSE response whose chunk boundaries deliberately fall mid-line. */
|
||||
function sseResponse(chunks: string[], status = 200): Response {
|
||||
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
|
||||
}
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
|
||||
interface Call {
|
||||
url: string;
|
||||
method?: string;
|
||||
headers: Record<string, string>;
|
||||
body: any;
|
||||
}
|
||||
let calls: Call[] = [];
|
||||
type Reply = Response | Error;
|
||||
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
|
||||
|
||||
function stubFetch(handler: Handler): void {
|
||||
calls = [];
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async (input: unknown, init?: RequestInit) => {
|
||||
const call: Call = {
|
||||
url: String(input),
|
||||
method: init?.method,
|
||||
headers: (init?.headers ?? {}) as Record<string, string>,
|
||||
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
|
||||
};
|
||||
calls.push(call);
|
||||
const r = await handler(call, calls.length - 1);
|
||||
if (r instanceof Error) throw r;
|
||||
return r;
|
||||
}),
|
||||
);
|
||||
}
|
||||
/** Replies with the given thunks in order (the last one repeats). */
|
||||
function queue(...items: Array<() => Reply>): Handler {
|
||||
return (_c, n) => items[Math.min(n, items.length - 1)]!();
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
|
||||
const out: T[] = [];
|
||||
for await (const x of it) out.push(x);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
|
||||
const events = await drain(adapter.stream(cfg, req));
|
||||
const done = events[events.length - 1];
|
||||
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
|
||||
return { events, done };
|
||||
}
|
||||
|
||||
async function failure(p: Promise<unknown>): Promise<AIError> {
|
||||
const err = await p.then(
|
||||
() => undefined,
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(err).toBeInstanceOf(AIError);
|
||||
return err as AIError;
|
||||
}
|
||||
|
||||
const TOOL: ToolDef = {
|
||||
name: 'lookup',
|
||||
description: 'Look things up',
|
||||
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
|
||||
};
|
||||
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
|
||||
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
|
||||
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
|
||||
role: 'assistant',
|
||||
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
|
||||
...extra,
|
||||
});
|
||||
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
|
||||
|
||||
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-adapter fixtures */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
|
||||
|
||||
interface Kit {
|
||||
name: string;
|
||||
adapter: ProviderAdapter;
|
||||
key: string;
|
||||
cfg: ProviderConfig;
|
||||
model: string;
|
||||
otherKind: ProviderKind;
|
||||
streamUrl: string;
|
||||
toolId: unknown;
|
||||
assertAuth(h: Record<string, string>): void;
|
||||
assertRequest(body: any): void; // system prompt + tool definitions
|
||||
assertHistory(body: any): void; // tool_call + tool_result mapping
|
||||
text(parts: string[]): string[];
|
||||
tool(frags: string[]): string[];
|
||||
usage(): { chunks: string[]; expected: Usage };
|
||||
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
|
||||
}
|
||||
|
||||
const openaiStyleErrors = (key: string): Kit['errors'] => ({
|
||||
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
|
||||
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
|
||||
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
|
||||
});
|
||||
|
||||
/* ---- Anthropic ---- */
|
||||
const anthropicKit: Kit = {
|
||||
name: 'anthropic',
|
||||
adapter: anthropicAdapter,
|
||||
key: 'sk-ant-TESTKEY-1234567890',
|
||||
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
|
||||
model: 'claude-sonnet-5-5',
|
||||
otherKind: 'openai',
|
||||
streamUrl: 'https://api.anthropic.com/v1/messages',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
|
||||
expect(h['anthropic-version']).toBe('2026-10-21');
|
||||
expect(h.Authorization).toBeUndefined();
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.system).toBe('SYS');
|
||||
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
|
||||
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.messages).toEqual([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
|
||||
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
|
||||
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
|
||||
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
|
||||
ev({ type: 'message_stop' }, 'message_stop'),
|
||||
],
|
||||
tool: (frags) => [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
|
||||
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
|
||||
ev({ type: 'content_block_stop', index: 0 }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
],
|
||||
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
],
|
||||
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
|
||||
}),
|
||||
errors: {
|
||||
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
|
||||
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
|
||||
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
|
||||
},
|
||||
};
|
||||
|
||||
/* ---- OpenAI (Responses API) ---- */
|
||||
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
|
||||
const openaiKit: Kit = {
|
||||
name: 'openai',
|
||||
adapter: openaiAdapter,
|
||||
key: 'sk-proj-TESTKEY-1234567890',
|
||||
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
|
||||
model: 'gpt-4o',
|
||||
otherKind: 'anthropic',
|
||||
streamUrl: 'https://api.openai.com/v1/responses',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.store).toBe(false);
|
||||
expect(b.instructions).toBe('SYS');
|
||||
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.input).toEqual([
|
||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
||||
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [
|
||||
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
|
||||
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
|
||||
],
|
||||
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
|
||||
tool: (frags) => [
|
||||
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
|
||||
ev({
|
||||
type: 'response.completed',
|
||||
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
|
||||
}),
|
||||
],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ev({ type: 'response.output_text.delta', delta: 'ok' }),
|
||||
ev({
|
||||
type: 'response.completed',
|
||||
response: {
|
||||
output: [oaMessage('ok')],
|
||||
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
|
||||
},
|
||||
}),
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
|
||||
};
|
||||
|
||||
/* ---- Gemini ---- */
|
||||
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
|
||||
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
|
||||
const geminiKit: Kit = {
|
||||
name: 'gemini',
|
||||
adapter: geminiAdapter,
|
||||
key: 'AIzaSyTESTKEY1234567890abcd',
|
||||
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
|
||||
model: 'gemini-3.8-flash',
|
||||
otherKind: 'anthropic',
|
||||
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
|
||||
toolId: expect.stringMatching(/^gem_/),
|
||||
assertAuth(h) {
|
||||
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
|
||||
expect(h.Authorization).toBeUndefined();
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
|
||||
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.contents).toEqual([
|
||||
{ role: 'user', parts: [{ text: 'hi' }] },
|
||||
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
|
||||
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
|
||||
// Gemini delivers a function call whole (args is an object), never as fragments.
|
||||
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: {
|
||||
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
|
||||
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
|
||||
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
|
||||
},
|
||||
};
|
||||
|
||||
/* ---- Chat Completions family ---- */
|
||||
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
|
||||
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
|
||||
return {
|
||||
...over,
|
||||
otherKind: 'gemini',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h.Authorization).toBe(`Bearer ${over.key}`);
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
|
||||
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.messages).toEqual([
|
||||
{ role: 'system', content: 'SYS' },
|
||||
{ role: 'user', content: 'hi' },
|
||||
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
|
||||
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
|
||||
tool: (frags) => [
|
||||
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
|
||||
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
|
||||
ccChunk({}, 'tool_calls'),
|
||||
DONE,
|
||||
],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ccChunk({ content: 'ok' }, 'stop'),
|
||||
ev({
|
||||
choices: [],
|
||||
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
|
||||
}),
|
||||
DONE,
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: openaiStyleErrors(over.key),
|
||||
};
|
||||
}
|
||||
const compatKit = chatCompletionsKit({
|
||||
name: 'openai_compatible',
|
||||
adapter: openaiCompatibleAdapter,
|
||||
key: 'sk-or-TESTKEY-1234567890',
|
||||
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
|
||||
model: 'meta/llama-3',
|
||||
streamUrl: 'https://router.example.com/api/v1/chat/completions',
|
||||
});
|
||||
const ollamaKit = chatCompletionsKit({
|
||||
name: 'ollama',
|
||||
adapter: ollamaAdapter,
|
||||
key: 'ollama-proxy-secret-1234',
|
||||
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
|
||||
model: 'llama3',
|
||||
streamUrl: 'http://localhost:11434/v1/chat/completions',
|
||||
});
|
||||
|
||||
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Shared behaviour across all five adapters */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe.each(kits)('$name adapter', (kit) => {
|
||||
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
|
||||
|
||||
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]!.url).toBe(kit.streamUrl);
|
||||
expect(calls[0]!.url).not.toContain(kit.key);
|
||||
expect(calls[0]!.method).toBe('POST');
|
||||
kit.assertAuth(calls[0]!.headers);
|
||||
kit.assertRequest(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('maps tool_call and tool_result history into the provider format', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
|
||||
kit.assertHistory(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
|
||||
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
|
||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
|
||||
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
|
||||
kit.assertHistory(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('streams text events in order and concatenates them in done', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
|
||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
|
||||
expect(done.message.role).toBe('assistant');
|
||||
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
|
||||
expect(done.stopReason).toBe('end');
|
||||
});
|
||||
|
||||
it('assembles a fragmented tool call into one tool_call event', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
|
||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
|
||||
expect(toolEvents).toHaveLength(1);
|
||||
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
||||
expect(events[events.length - 1]).toBe(done);
|
||||
});
|
||||
|
||||
it('maps usage onto the neutral convention', async () => {
|
||||
const { chunks, expected } = kit.usage();
|
||||
stubFetch(queue(() => sseResponse(chunks)));
|
||||
const { done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(done.usage).toEqual(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['auth', 'auth', 401],
|
||||
['rate', 'rate_limit', 429],
|
||||
['ctx', 'context_length', 400],
|
||||
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
|
||||
const [st, body] = kit.errors[which];
|
||||
stubFetch(queue(() => jsonResponse(body, st)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.code).toBe(code);
|
||||
expect(err.status).toBe(status);
|
||||
expect(err.message).not.toContain(kit.key);
|
||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
||||
});
|
||||
|
||||
it('scrubs the API key from provider text', async () => {
|
||||
const [st, body] = kit.errors.auth;
|
||||
stubFetch(queue(() => jsonResponse(body, st)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.providerMessage).toContain('[redacted]');
|
||||
expect(JSON.stringify(err)).not.toContain(kit.key);
|
||||
});
|
||||
|
||||
it('maps a fetch TypeError to a network error without leaking the key', async () => {
|
||||
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.code).toBe('network');
|
||||
expect(err.retryable).toBe(true);
|
||||
expect(err.message).not.toContain(kit.key);
|
||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Anthropic specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('anthropic providerData', () => {
|
||||
const thinkingTurn = [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
|
||||
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
|
||||
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
|
||||
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
|
||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
|
||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
];
|
||||
const replayed = [
|
||||
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
|
||||
{ type: 'text', text: 'Sure' },
|
||||
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
|
||||
];
|
||||
const followUp = (assistant: ChatMessage): ChatRequest => ({
|
||||
model: 'claude-sonnet-5-5',
|
||||
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
|
||||
});
|
||||
|
||||
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
|
||||
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
|
||||
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
|
||||
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
|
||||
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
|
||||
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
|
||||
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
|
||||
|
||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
|
||||
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
|
||||
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
|
||||
});
|
||||
|
||||
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
|
||||
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
|
||||
() => sseResponse(anthropicKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
|
||||
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('anthropic listModels', () => {
|
||||
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() =>
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: 'claude-haiku-4-5',
|
||||
display_name: 'Haiku API Name',
|
||||
max_input_tokens: 200000,
|
||||
max_tokens: 64000,
|
||||
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
|
||||
},
|
||||
{ id: 'claude-embed-1' },
|
||||
],
|
||||
has_more: true,
|
||||
last_id: 'claude-haiku-4-5',
|
||||
}),
|
||||
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
|
||||
),
|
||||
);
|
||||
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
|
||||
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
|
||||
expect(calls[0]!.url).not.toContain(anthropicKit.key);
|
||||
anthropicKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
|
||||
expect(models[0]).toMatchObject({
|
||||
label: 'Haiku API Name', // reported by the API, wins over the catalog
|
||||
contextWindow: 200000,
|
||||
maxOutput: 64000,
|
||||
priceIn: 1, // from the catalog
|
||||
priceOut: 5,
|
||||
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
|
||||
});
|
||||
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('openai providerData and reasoning', () => {
|
||||
const output = [
|
||||
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
|
||||
oaMessage('Hi'),
|
||||
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
||||
];
|
||||
const turn = [
|
||||
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
|
||||
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
|
||||
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
|
||||
];
|
||||
|
||||
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
|
||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
|
||||
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
|
||||
const body = calls[0]!.body;
|
||||
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
|
||||
expect(body.include).toEqual(['reasoning.encrypted_content']);
|
||||
expect(body.temperature).toBeUndefined(); // reasoning models reject it
|
||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
|
||||
|
||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
|
||||
expect(calls[1]!.body.input).toEqual([
|
||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
||||
...output,
|
||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('retries once without reasoning.effort when the model rejects it', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
|
||||
() => sseResponse(openaiKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
|
||||
expect(calls[0]!.body.reasoning.effort).toBe('none');
|
||||
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
|
||||
});
|
||||
});
|
||||
|
||||
describe('openai listModels', () => {
|
||||
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
|
||||
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
|
||||
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
|
||||
const models = await openaiAdapter.listModels(openaiKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
|
||||
openaiKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
|
||||
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
|
||||
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Gemini specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('gemini providerData and request config', () => {
|
||||
const turn = [
|
||||
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
|
||||
gemChunk([{ text: 'Hello' }]),
|
||||
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
|
||||
];
|
||||
const replayed = [
|
||||
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
|
||||
{ text: 'Hello' },
|
||||
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
|
||||
];
|
||||
|
||||
it('keeps thought signatures in providerData and replays the raw parts', async () => {
|
||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
|
||||
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
|
||||
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
|
||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
|
||||
|
||||
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
|
||||
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
|
||||
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
|
||||
});
|
||||
|
||||
it('turns a blocked prompt into a content_filter error', async () => {
|
||||
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
|
||||
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
|
||||
expect(err.code).toBe('content_filter');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gemini listModels', () => {
|
||||
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() =>
|
||||
jsonResponse({
|
||||
models: [
|
||||
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
|
||||
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
|
||||
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
|
||||
],
|
||||
nextPageToken: 'tok1',
|
||||
}),
|
||||
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
|
||||
),
|
||||
);
|
||||
const models = await geminiAdapter.listModels(geminiKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
|
||||
expect(calls[1]!.url).toContain('pageToken=tok1');
|
||||
for (const c of calls) {
|
||||
expect(c.url).not.toContain(geminiKit.key);
|
||||
geminiKit.assertAuth(c.headers);
|
||||
}
|
||||
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
|
||||
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
|
||||
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('toGeminiSchema', () => {
|
||||
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
|
||||
const out = toGeminiSchema({
|
||||
$schema: 'http://json-schema.org/draft-07/schema#',
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
|
||||
when: { type: 'string', format: 'date-time' },
|
||||
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
|
||||
mode: { enum: ['a', 'b'] },
|
||||
nested: { type: 'object' },
|
||||
},
|
||||
required: ['name'],
|
||||
});
|
||||
expect(out).toEqual({
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string', nullable: true },
|
||||
when: { type: 'string', format: 'date-time' },
|
||||
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
|
||||
mode: { enum: ['a', 'b'], type: 'string' },
|
||||
nested: { type: 'object', properties: {} },
|
||||
},
|
||||
required: ['name'],
|
||||
});
|
||||
});
|
||||
it('falls back to a string schema for non-objects', () => {
|
||||
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI-compatible and Ollama specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('openai-compatible stream_options fallback', () => {
|
||||
it('retries without stream_options when the first 400 complains about it', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
|
||||
() => sseResponse(compatKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
|
||||
expect(calls[1]!.body.stream_options).toBeUndefined();
|
||||
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
|
||||
});
|
||||
|
||||
it('does not write providerData (it only speaks neutral content)', async () => {
|
||||
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
|
||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
||||
expect(done.message.providerData).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('openai-compatible listModels', () => {
|
||||
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(() =>
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: 'anthropic/claude-sonnet-5-5',
|
||||
name: 'Claude Sonnet 5.5',
|
||||
context_length: 1000000,
|
||||
top_provider: { max_completion_tokens: 64000 },
|
||||
pricing: { prompt: '0.000002', completion: '0.00001' },
|
||||
supported_parameters: ['tools', 'reasoning'],
|
||||
architecture: { input_modalities: ['text', 'image'] },
|
||||
},
|
||||
{ id: 'openai/text-embedding-3-small' },
|
||||
{ id: 'vendor/plain' },
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
|
||||
compatKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
|
||||
expect(models[0]).toMatchObject({
|
||||
label: 'Claude Sonnet 5.5',
|
||||
contextWindow: 1000000,
|
||||
maxOutput: 64000,
|
||||
priceIn: 2,
|
||||
priceOut: 10,
|
||||
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
|
||||
});
|
||||
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
|
||||
expect(models[1]).toEqual({ id: 'vendor/plain' });
|
||||
});
|
||||
|
||||
it('reports "unsupported" when the server has no /models endpoint', async () => {
|
||||
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
|
||||
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
|
||||
expect(err.code).toBe('unsupported');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ollama listModels', () => {
|
||||
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
|
||||
stubFetch((c) => {
|
||||
if (c.url.endsWith('/api/tags')) {
|
||||
return jsonResponse({
|
||||
models: [
|
||||
{ name: 'old:1b', model: 'old:1b' },
|
||||
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
|
||||
{ name: 'llama3:latest', model: 'llama3:latest' },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (c.url.endsWith('/api/show')) {
|
||||
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
|
||||
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
|
||||
return jsonResponse({ error: 'unsupported' }, 404);
|
||||
}
|
||||
return new Response('?', { status: 404 });
|
||||
});
|
||||
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
|
||||
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
|
||||
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
|
||||
ollamaKit.assertAuth(calls[0]!.headers);
|
||||
expect(models).toEqual([
|
||||
{
|
||||
id: 'llama3:latest',
|
||||
label: 'llama3:latest',
|
||||
priceIn: 0,
|
||||
priceOut: 0,
|
||||
contextWindow: 8192,
|
||||
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
|
||||
},
|
||||
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('strips /v1 from a configured base URL', async () => {
|
||||
stubFetch(queue(() => jsonResponse({ models: [] })));
|
||||
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
|
||||
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeCompatBase', () => {
|
||||
it.each([
|
||||
['https://api.x.com/v1', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/', 'https://api.x.com/v1'],
|
||||
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
|
||||
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
|
||||
['http://localhost:1234', 'http://localhost:1234/v1'],
|
||||
['http://localhost:1234/', 'http://localhost:1234/v1'],
|
||||
])('%s -> %s', (input, expected) => {
|
||||
expect(normalizeCompatBase(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* catalog */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('lookupModelDefaults', () => {
|
||||
it('matches exact ids', () => {
|
||||
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
|
||||
});
|
||||
it('matches dated snapshots by prefix, without the base label', () => {
|
||||
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
|
||||
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
|
||||
expect(d.label).toBeUndefined();
|
||||
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
|
||||
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
|
||||
});
|
||||
it('keeps capabilities but drops prices for a different variant of a known model', () => {
|
||||
const d = lookupModelDefaults('openai', 'o3-deep-research');
|
||||
expect(d.capabilities?.reasoning).toBe(true);
|
||||
expect(d.priceIn).toBeUndefined();
|
||||
expect(d.priceOut).toBeUndefined();
|
||||
});
|
||||
it('returns {} for unknown models and for kinds without a table', () => {
|
||||
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
|
||||
expect(lookupModelDefaults('anthropic', '')).toEqual({});
|
||||
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
|
||||
});
|
||||
it('borrows capabilities but not prices or temperature for router ids', () => {
|
||||
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
|
||||
expect(d.priceIn).toBeUndefined();
|
||||
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
|
||||
expect(d.capabilities).not.toHaveProperty('temperature');
|
||||
});
|
||||
it('returns copies that are safe to mutate', () => {
|
||||
const a = lookupModelDefaults('openai', 'gpt-4o');
|
||||
a.capabilities!.tools = false;
|
||||
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* parseSSE */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('parseSSE', () => {
|
||||
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
|
||||
|
||||
it('handles CRLF line endings', async () => {
|
||||
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
|
||||
});
|
||||
it('joins multi-line data with newlines', async () => {
|
||||
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
|
||||
});
|
||||
it('ignores comments and keep-alives', async () => {
|
||||
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
|
||||
});
|
||||
it('stops at [DONE]', async () => {
|
||||
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
|
||||
});
|
||||
it('handles chunk boundaries inside a line', async () => {
|
||||
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
|
||||
});
|
||||
it('handles CRLF split across chunks', async () => {
|
||||
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
||||
});
|
||||
it('handles bare CR line endings', async () => {
|
||||
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
||||
});
|
||||
it('flushes a final event with no trailing blank line', async () => {
|
||||
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
|
||||
});
|
||||
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
|
||||
const bytes = enc.encode('data: €uro\n\n');
|
||||
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
|
||||
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
|
||||
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue