Study/server/ai/providers/providers.test.ts
Elijah 3e74ecd843 Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 08:16:41 -07:00

932 lines
45 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/* eslint-disable @typescript-eslint/no-explicit-any */
/**
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
* Wire shapes below are derived from what each adapter's parser reads.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
import { anthropicAdapter } from './anthropic.ts';
import { lookupModelDefaults } from './catalog.ts';
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
import { parseSSE } from './http.ts';
import { ollamaAdapter } from './ollama.ts';
import { openaiAdapter } from './openai.ts';
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
const enc = new TextEncoder();
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
return new ReadableStream({
start(c) {
for (const ch of chunks) c.enqueue(ch);
c.close();
},
});
}
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
/** One SSE event. */
function ev(data: unknown, event?: string): string {
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
}
const DONE = 'data: [DONE]\n\n';
function rechunk(s: string, n: number): string[] {
const out: string[] = [];
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
return out;
}
/** SSE response whose chunk boundaries deliberately fall mid-line. */
function sseResponse(chunks: string[], status = 200): Response {
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
}
function jsonResponse(body: unknown, status = 200): Response {
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
}
interface Call {
url: string;
method?: string;
headers: Record<string, string>;
body: any;
}
let calls: Call[] = [];
type Reply = Response | Error;
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
function stubFetch(handler: Handler): void {
calls = [];
vi.stubGlobal(
'fetch',
vi.fn(async (input: unknown, init?: RequestInit) => {
const call: Call = {
url: String(input),
method: init?.method,
headers: (init?.headers ?? {}) as Record<string, string>,
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
};
calls.push(call);
const r = await handler(call, calls.length - 1);
if (r instanceof Error) throw r;
return r;
}),
);
}
/** Replies with the given thunks in order (the last one repeats). */
function queue(...items: Array<() => Reply>): Handler {
return (_c, n) => items[Math.min(n, items.length - 1)]!();
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
const out: T[] = [];
for await (const x of it) out.push(x);
return out;
}
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
const events = await drain(adapter.stream(cfg, req));
const done = events[events.length - 1];
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
return { events, done };
}
async function failure(p: Promise<unknown>): Promise<AIError> {
const err = await p.then(
() => undefined,
(e: unknown) => e,
);
expect(err).toBeInstanceOf(AIError);
return err as AIError;
}
const TOOL: ToolDef = {
name: 'lookup',
description: 'Look things up',
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
};
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
role: 'assistant',
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
...extra,
});
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
/* ------------------------------------------------------------------ */
/* Per-adapter fixtures */
/* ------------------------------------------------------------------ */
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
interface Kit {
name: string;
adapter: ProviderAdapter;
key: string;
cfg: ProviderConfig;
model: string;
otherKind: ProviderKind;
streamUrl: string;
toolId: unknown;
assertAuth(h: Record<string, string>): void;
assertRequest(body: any): void; // system prompt + tool definitions
assertHistory(body: any): void; // tool_call + tool_result mapping
text(parts: string[]): string[];
tool(frags: string[]): string[];
usage(): { chunks: string[]; expected: Usage };
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
}
const openaiStyleErrors = (key: string): Kit['errors'] => ({
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
});
/* ---- Anthropic ---- */
const anthropicKit: Kit = {
name: 'anthropic',
adapter: anthropicAdapter,
key: 'sk-ant-TESTKEY-1234567890',
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
model: 'claude-sonnet-5-5',
otherKind: 'openai',
streamUrl: 'https://api.anthropic.com/v1/messages',
toolId: 'call_1',
assertAuth(h) {
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
expect(h['anthropic-version']).toBe('2026-10-21');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.system).toBe('SYS');
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
]);
},
text: (parts) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
ev({ type: 'message_stop' }, 'message_stop'),
],
tool: (frags) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
ev({ type: 'content_block_stop', index: 0 }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
],
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
usage: () => ({
chunks: [
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
ev({ type: 'message_stop' }),
],
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
}),
errors: {
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
},
};
/* ---- OpenAI (Responses API) ---- */
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
const openaiKit: Kit = {
name: 'openai',
adapter: openaiAdapter,
key: 'sk-proj-TESTKEY-1234567890',
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
model: 'gpt-4o',
otherKind: 'anthropic',
streamUrl: 'https://api.openai.com/v1/responses',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.store).toBe(false);
expect(b.instructions).toBe('SYS');
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
},
assertHistory(b) {
expect(b.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
},
text: (parts) => [
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
],
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
tool: (frags) => [
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
ev({
type: 'response.completed',
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
}),
],
usage: () => ({
chunks: [
ev({ type: 'response.output_text.delta', delta: 'ok' }),
ev({
type: 'response.completed',
response: {
output: [oaMessage('ok')],
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
},
}),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
};
/* ---- Gemini ---- */
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
const geminiKit: Kit = {
name: 'gemini',
adapter: geminiAdapter,
key: 'AIzaSyTESTKEY1234567890abcd',
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
model: 'gemini-3.8-flash',
otherKind: 'anthropic',
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
toolId: expect.stringMatching(/^gem_/),
assertAuth(h) {
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
},
assertHistory(b) {
expect(b.contents).toEqual([
{ role: 'user', parts: [{ text: 'hi' }] },
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
]);
},
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
// Gemini delivers a function call whole (args is an object), never as fragments.
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
usage: () => ({
chunks: [
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: {
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
},
};
/* ---- Chat Completions family ---- */
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
return {
...over,
otherKind: 'gemini',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe(`Bearer ${over.key}`);
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'hi' },
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
]);
},
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
tool: (frags) => [
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
ccChunk({}, 'tool_calls'),
DONE,
],
usage: () => ({
chunks: [
ccChunk({ content: 'ok' }, 'stop'),
ev({
choices: [],
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
}),
DONE,
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors(over.key),
};
}
const compatKit = chatCompletionsKit({
name: 'openai_compatible',
adapter: openaiCompatibleAdapter,
key: 'sk-or-TESTKEY-1234567890',
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
model: 'meta/llama-3',
streamUrl: 'https://router.example.com/api/v1/chat/completions',
});
const ollamaKit = chatCompletionsKit({
name: 'ollama',
adapter: ollamaAdapter,
key: 'ollama-proxy-secret-1234',
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
model: 'llama3',
streamUrl: 'http://localhost:11434/v1/chat/completions',
});
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
/* ------------------------------------------------------------------ */
/* Shared behaviour across all five adapters */
/* ------------------------------------------------------------------ */
describe.each(kits)('$name adapter', (kit) => {
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, baseReq());
expect(calls).toHaveLength(1);
expect(calls[0]!.url).toBe(kit.streamUrl);
expect(calls[0]!.url).not.toContain(kit.key);
expect(calls[0]!.method).toBe('POST');
kit.assertAuth(calls[0]!.headers);
kit.assertRequest(calls[0]!.body);
});
it('maps tool_call and tool_result history into the provider format', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
kit.assertHistory(calls[0]!.body);
});
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
kit.assertHistory(calls[0]!.body);
});
it('streams text events in order and concatenates them in done', async () => {
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
expect(done.message.role).toBe('assistant');
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
expect(done.stopReason).toBe('end');
});
it('assembles a fragmented tool call into one tool_call event', async () => {
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
expect(toolEvents).toHaveLength(1);
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(done.stopReason).toBe('tool_use');
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(events[events.length - 1]).toBe(done);
});
it('maps usage onto the neutral convention', async () => {
const { chunks, expected } = kit.usage();
stubFetch(queue(() => sseResponse(chunks)));
const { done } = await run(kit.adapter, kit.cfg, baseReq());
expect(done.usage).toEqual(expected);
});
it.each([
['auth', 'auth', 401],
['rate', 'rate_limit', 429],
['ctx', 'context_length', 400],
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
const [st, body] = kit.errors[which];
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe(code);
expect(err.status).toBe(status);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
it('scrubs the API key from provider text', async () => {
const [st, body] = kit.errors.auth;
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.providerMessage).toContain('[redacted]');
expect(JSON.stringify(err)).not.toContain(kit.key);
});
it('maps a fetch TypeError to a network error without leaking the key', async () => {
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe('network');
expect(err.retryable).toBe(true);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
});
/* ------------------------------------------------------------------ */
/* Anthropic specifics */
/* ------------------------------------------------------------------ */
describe('anthropic providerData', () => {
const thinkingTurn = [
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
];
const replayed = [
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
{ type: 'text', text: 'Sure' },
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
];
const followUp = (assistant: ChatMessage): ChatRequest => ({
model: 'claude-sonnet-5-5',
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
});
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
});
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
stubFetch(
queue(
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
() => sseResponse(anthropicKit.text(['ok'])),
),
);
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
expect(calls).toHaveLength(2);
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
});
});
describe('anthropic listModels', () => {
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
data: [
{
id: 'claude-haiku-4-5',
display_name: 'Haiku API Name',
max_input_tokens: 200000,
max_tokens: 64000,
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
},
{ id: 'claude-embed-1' },
],
has_more: true,
last_id: 'claude-haiku-4-5',
}),
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
),
);
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
expect(calls[0]!.url).not.toContain(anthropicKit.key);
anthropicKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
expect(models[0]).toMatchObject({
label: 'Haiku API Name', // reported by the API, wins over the catalog
contextWindow: 200000,
maxOutput: 64000,
priceIn: 1, // from the catalog
priceOut: 5,
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
});
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI specifics */
/* ------------------------------------------------------------------ */
describe('openai providerData and reasoning', () => {
const output = [
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
oaMessage('Hi'),
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
];
const turn = [
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
];
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
const body = calls[0]!.body;
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
expect(body.include).toEqual(['reasoning.encrypted_content']);
expect(body.temperature).toBeUndefined(); // reasoning models reject it
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
...output,
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
});
it('retries once without reasoning.effort when the model rejects it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
() => sseResponse(openaiKit.text(['ok'])),
),
);
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
expect(calls[0]!.body.reasoning.effort).toBe('none');
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
});
});
describe('openai listModels', () => {
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
const models = await openaiAdapter.listModels(openaiKit.cfg);
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
openaiKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
});
});
/* ------------------------------------------------------------------ */
/* Gemini specifics */
/* ------------------------------------------------------------------ */
describe('gemini providerData and request config', () => {
const turn = [
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
gemChunk([{ text: 'Hello' }]),
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
];
const replayed = [
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
{ text: 'Hello' },
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
];
it('keeps thought signatures in providerData and replays the raw parts', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
});
it('turns a blocked prompt into a content_filter error', async () => {
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
expect(err.code).toBe('content_filter');
});
});
describe('gemini listModels', () => {
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
models: [
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
],
nextPageToken: 'tok1',
}),
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
),
);
const models = await geminiAdapter.listModels(geminiKit.cfg);
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
expect(calls[1]!.url).toContain('pageToken=tok1');
for (const c of calls) {
expect(c.url).not.toContain(geminiKit.key);
geminiKit.assertAuth(c.headers);
}
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
});
});
describe('toGeminiSchema', () => {
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
const out = toGeminiSchema({
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
additionalProperties: false,
properties: {
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
mode: { enum: ['a', 'b'] },
nested: { type: 'object' },
},
required: ['name'],
});
expect(out).toEqual({
type: 'object',
properties: {
name: { type: 'string', nullable: true },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
mode: { enum: ['a', 'b'], type: 'string' },
nested: { type: 'object', properties: {} },
},
required: ['name'],
});
});
it('falls back to a string schema for non-objects', () => {
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI-compatible and Ollama specifics */
/* ------------------------------------------------------------------ */
describe('openai-compatible stream_options fallback', () => {
it('retries without stream_options when the first 400 complains about it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
() => sseResponse(compatKit.text(['ok'])),
),
);
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(calls).toHaveLength(2);
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
expect(calls[1]!.body.stream_options).toBeUndefined();
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
});
it('does not write providerData (it only speaks neutral content)', async () => {
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(done.message.providerData).toBeUndefined();
});
});
describe('openai-compatible listModels', () => {
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
stubFetch(
queue(() =>
jsonResponse({
data: [
{
id: 'anthropic/claude-sonnet-5-5',
name: 'Claude Sonnet 5.5',
context_length: 1000000,
top_provider: { max_completion_tokens: 64000 },
pricing: { prompt: '0.000002', completion: '0.00001' },
supported_parameters: ['tools', 'reasoning'],
architecture: { input_modalities: ['text', 'image'] },
},
{ id: 'openai/text-embedding-3-small' },
{ id: 'vendor/plain' },
],
}),
),
);
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
compatKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
expect(models[0]).toMatchObject({
label: 'Claude Sonnet 5.5',
contextWindow: 1000000,
maxOutput: 64000,
priceIn: 2,
priceOut: 10,
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
});
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
expect(models[1]).toEqual({ id: 'vendor/plain' });
});
it('reports "unsupported" when the server has no /models endpoint', async () => {
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
expect(err.code).toBe('unsupported');
});
});
describe('ollama listModels', () => {
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
stubFetch((c) => {
if (c.url.endsWith('/api/tags')) {
return jsonResponse({
models: [
{ name: 'old:1b', model: 'old:1b' },
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
{ name: 'llama3:latest', model: 'llama3:latest' },
],
});
}
if (c.url.endsWith('/api/show')) {
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
return jsonResponse({ error: 'unsupported' }, 404);
}
return new Response('?', { status: 404 });
});
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
ollamaKit.assertAuth(calls[0]!.headers);
expect(models).toEqual([
{
id: 'llama3:latest',
label: 'llama3:latest',
priceIn: 0,
priceOut: 0,
contextWindow: 8192,
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
},
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
]);
});
it('strips /v1 from a configured base URL', async () => {
stubFetch(queue(() => jsonResponse({ models: [] })));
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
});
});
describe('normalizeCompatBase', () => {
it.each([
['https://api.x.com/v1', 'https://api.x.com/v1'],
['https://api.x.com/v1/', 'https://api.x.com/v1'],
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
['http://localhost:1234', 'http://localhost:1234/v1'],
['http://localhost:1234/', 'http://localhost:1234/v1'],
])('%s -> %s', (input, expected) => {
expect(normalizeCompatBase(input)).toBe(expected);
});
});
/* ------------------------------------------------------------------ */
/* catalog */
/* ------------------------------------------------------------------ */
describe('lookupModelDefaults', () => {
it('matches exact ids', () => {
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
});
it('matches dated snapshots by prefix, without the base label', () => {
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
expect(d.label).toBeUndefined();
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
});
it('keeps capabilities but drops prices for a different variant of a known model', () => {
const d = lookupModelDefaults('openai', 'o3-deep-research');
expect(d.capabilities?.reasoning).toBe(true);
expect(d.priceIn).toBeUndefined();
expect(d.priceOut).toBeUndefined();
});
it('returns {} for unknown models and for kinds without a table', () => {
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
expect(lookupModelDefaults('anthropic', '')).toEqual({});
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
});
it('borrows capabilities but not prices or temperature for router ids', () => {
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
expect(d.priceIn).toBeUndefined();
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
expect(d.capabilities).not.toHaveProperty('temperature');
});
it('returns copies that are safe to mutate', () => {
const a = lookupModelDefaults('openai', 'gpt-4o');
a.capabilities!.tools = false;
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
});
});
/* ------------------------------------------------------------------ */
/* parseSSE */
/* ------------------------------------------------------------------ */
describe('parseSSE', () => {
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
it('handles CRLF line endings', async () => {
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
});
it('joins multi-line data with newlines', async () => {
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
});
it('ignores comments and keep-alives', async () => {
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
});
it('stops at [DONE]', async () => {
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
});
it('handles chunk boundaries inside a line', async () => {
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
});
it('handles CRLF split across chunks', async () => {
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('handles bare CR line endings', async () => {
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('flushes a final event with no trailing blank line', async () => {
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
});
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
const bytes = enc.encode('data: €uro\n\n');
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
});
});