Study/server/ai/providers/openaiCompatible.ts
Elijah 25bc03ea75
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 29s
Fourth revision: OpenRouter, live tool stages, Explain layout, share sections
- Interactive blocks are encouraged by default; Explain gets a reduced library
  and never writes practice questions.
- Assistant tool steps breathe while running, show live stages from generation
  and the Python check, and read in the past tense when done.
- The assistant asks before searching the web when class materials do not
  cover a question; clarifying questions list every class's exams and never
  ask about format (always NCLEX).
- Explain opens beside the question from 900px wide, moving the question left
  while open and back on the next one.
- Generate dialog drops the practice set/exam switch (presets cover it) and the
  Generate buttons lose their icon.
- Python check notes are green when every check passed, applied fixes neutral.
- Share pages group by exam, then Quizzes, Flashcards, Notes, Class Materials,
  hiding empty groups; the class page uses the same order. "Documents" is now
  "Class Materials".
- OpenRouter provider replaces the Ollama type; migration 6 turns Ollama
  connections into OpenAI-compatible ones at /v1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 21:54:39 -07:00

428 lines
21 KiB
TypeScript

/**
* Generic Chat Completions (`POST {base}/chat/completions`) for Groq, Together, LM Studio,
* vLLM, llama.cpp, Ollama and friends, and the engine behind the DeepSeek and OpenRouter
* adapters.
*
* These servers agree on the basics and disagree on everything else, so this
* adapter is deliberately forgiving:
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
* once when a server answers 400 about them;
* - usage may be missing, and may arrive on a chunk with no choices;
* - tool-call argument fragments may arrive in any chunking, with or without ids;
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
* - the last chunk may lack a `finish_reason`.
*/
import { type EmbedRequest, type EmbedResult,
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type ProviderKind,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { parseEmbeddings, dataUrl, isChatModelId, makeUsage, nearestEffort, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
export type Json = Record<string, unknown>;
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
export function normalizeCompatBase(url: string): string {
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
try {
const u = new URL(base);
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
} catch {
/* validated by the caller */
}
return base;
}
export interface CompatOptions {
kind: ProviderKind;
label: string;
baseUrl: string;
apiKey: string | null;
headers?: Record<string, string>;
/**
* Server-specific request tweaks (DeepSeek's `thinking` switch, its effort levels...).
* Called on the finished body before it is sent.
*/
adjustBody?: (body: Json, req: ChatRequest) => void;
/**
* The server requires the reasoning text of earlier assistant turns to be sent back
* (`reasoning_content`), as DeepSeek does in thinking mode with tools.
*/
replayReasoning?: boolean;
/**
* Whether `required` / named tool choices may be sent for this request. When false the
* tool stays available with `auto` and the system prompt asks for the call instead.
*/
forcedToolChoice?: (req: ChatRequest) => boolean;
}
/** Levels sent as `reasoning_effort` by default. "max" has no OpenAI-style equivalent. */
const COMPAT_EFFORTS = ['minimal', 'low', 'medium', 'high', 'xhigh'] as const;
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
const blocks: Json[] = [];
for (const p of parts) {
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
else if (p.type === 'image') {
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
} else if (p.type === 'file') {
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
// Chat Completions "file" parts (OpenAI, OpenRouter and others) for models marked as reading PDFs.
else if (p.mime === 'application/pdf' && caps.pdf) blocks.push({ type: 'file', file: { filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) } });
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
// Plain strings are understood by every server; arrays only by multimodal ones.
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
return blocks;
}
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system', replay?: { kind: ProviderKind }): Json[] {
const caps = req.capabilities ?? {};
const out: Json[] = [];
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
for (const msg of req.messages) {
if (msg.role === 'assistant') {
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
if (!text && calls.length === 0) continue;
const wire: Json = { role: 'assistant', content: text || null };
if (replay) {
// Same server: hand back its reasoning verbatim. A turn written by another provider
// has none, which these servers accept as an empty string.
const pd = msg.providerData;
wire.reasoning_content = pd?.kind === replay.kind && isRecord(pd.data) && typeof pd.data.reasoning_content === 'string' ? pd.data.reasoning_content : '';
}
if (calls.length > 0) {
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
}
out.push(wire);
continue;
}
// Tool results must directly follow the assistant turn that requested them.
for (const p of msg.content) {
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
}
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
if (rest.length > 0) {
const content = userContent(rest, caps);
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
}
}
return out;
}
interface BodyFlags {
tools: boolean;
/** Send `required` / named tool choices as such; false = ask for the call in the prompt instead. */
forcedChoice: boolean;
usage: boolean;
jsonMode: boolean;
temperature: boolean;
reasoning: boolean;
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
}
function forcedToolNote(choice: ChatRequest['toolChoice']): string | null {
if (choice === 'required') return 'For this turn you must call one of the available tools. Begin your response with the tool call.';
if (isRecord(choice)) return `For this turn you must call the \`${choice.name}\` tool. Begin your response with that tool call and do not answer in plain text.`;
return null;
}
function buildBody(req: ChatRequest, flags: BodyFlags, o: CompatOptions): Json {
const caps = req.capabilities ?? {};
const sendTools = flags.tools && caps.tools !== false && Boolean(req.tools?.length);
const note = sendTools && !flags.forcedChoice ? forcedToolNote(req.toolChoice) : null;
const effective = note ? { ...req, system: [req.system?.trim(), note].filter(Boolean).join('\n\n') } : req;
const body: Json = { model: req.model, messages: toChatMessages(effective, 'system', o.replayReasoning ? { kind: o.kind } : undefined), stream: true };
if (flags.usage) body.stream_options = { include_usage: true };
if (sendTools) {
body.tools = req.tools!.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
const choice = req.toolChoice;
if (choice === 'none' || choice === 'auto') body.tool_choice = choice;
else if (choice === 'required') body.tool_choice = flags.forcedChoice ? 'required' : 'auto';
else if (isRecord(choice)) body.tool_choice = flags.forcedChoice ? { type: 'function', function: { name: choice.name } } : 'auto';
}
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = nearestEffort(req.reasoningEffort, COMPAT_EFFORTS);
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
if (o.adjustBody) {
o.adjustBody(body, req);
if (!flags.reasoning) delete body.reasoning_effort;
}
return body;
}
/** A 400 that says the model cannot do tool calling at all (not just a rejected parameter). */
const NO_TOOLS_RE = /(does not|doesn't|do not|cannot|can't|not) support(s|ed)? (tool|function)|(tool|function)[ _-]?(call(ing|s)?|use)? (is|are) not (supported|available|enabled)|unsupported.{0,20}(tool|function)|no tool support/;
function mapFinish(reason: string): StopReason {
switch (reason) {
case 'stop':
case 'eos':
case 'end_turn':
return 'end';
case 'tool_calls':
case 'function_call':
return 'tool_use';
case 'length':
case 'max_tokens':
return 'max_tokens';
case 'content_filter':
return 'refusal';
default:
return 'other';
}
}
function authHeaders(o: CompatOptions): Record<string, string> {
const h: Record<string, string> = { ...o.headers };
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
return h;
}
function errorContext(o: CompatOptions): ErrorContext {
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
}
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(o);
const base = normalizeCompatBase(o.baseUrl);
const local = isLocalUrl(base);
const flags: BodyFlags = { tools: true, forcedChoice: o.forcedToolChoice ? o.forcedToolChoice(req) : true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
const forcing = req.toolChoice === 'required' || isRecord(req.toolChoice);
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(`${base}/chat/completions`, {
...ctx,
headers: authHeaders(o),
body: buildBody(req, flags, o),
signal: req.signal,
// A local server may spend minutes loading a model into memory before the first byte.
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
});
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
// A forced tool choice is often what is rejected (DeepSeek in thinking mode, some
// routers): keep the tools and ask for the call in the prompt instead.
else if (flags.forcedChoice && forcing && flags.tools && req.tools?.length && /tool_choice|tool choice|required|function_call|named/.test(detail)) flags.forcedChoice = false;
else if (flags.tools && req.tools?.length && NO_TOOLS_RE.test(detail)) {
// The model (or server) cannot do tool calling at all.
if (forcing) {
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
status: err.status,
providerMessage: err.providerMessage,
});
}
flags.tools = false;
} else throw err;
}
}
let text = '';
let finish = '';
let reasoningSeen = false;
let reasoningText = '';
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
const calls: { id: string; name: string; args: string }[] = [];
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
if (isRecord(data.usage)) {
const u = data.usage;
usage.input = num(u.prompt_tokens) || usage.input;
usage.output = num(u.completion_tokens) || usage.output;
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
// DeepSeek's own field (the same number as cached_tokens where both are sent).
if (!usage.cacheRead && num(u.prompt_cache_hit_tokens)) usage.cacheRead = num(u.prompt_cache_hit_tokens);
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
}
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
if (!isRecord(choice)) continue;
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
if (!delta) continue;
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
if (reasoning) {
reasoningSeen = true;
if (o.replayReasoning) reasoningText += reasoning;
yield { type: 'reasoning', delta: reasoning };
}
const piece = typeof delta.content === 'string' ? delta.content : '';
if (piece) {
text += piece;
yield { type: 'text', delta: piece };
}
if (Array.isArray(delta.tool_calls)) {
for (const raw of delta.tool_calls) {
if (!isRecord(raw)) continue;
const fn = isRecord(raw.function) ? raw.function : {};
// `index` identifies the call; servers that omit it send one whole call per entry.
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
let slot = index >= 0 ? calls[index] : undefined;
if (!slot) {
slot = { id: '', name: '', args: '' };
if (index >= 0) calls[index] = slot;
else calls.push(slot);
}
if (str(raw.id)) slot.id = str(raw.id);
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
}
}
}
const stop = finish ? mapFinish(finish) : 'end';
const content: ContentPart[] = [];
if (text) content.push({ type: 'text', text });
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
for (const c of calls) {
if (!c || !c.name) continue;
let input = parseArgsStrict(c.args);
if (!input) {
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
input = parseToolArguments(c.args, o.label, c.name);
}
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
}
for (const call of done) {
content.push({ type: 'tool_call', ...call });
yield { type: 'tool_call', call };
}
if (content.length === 0 && !reasoningSeen && !finish) {
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
}
const message: ChatMessage = { role: 'assistant', content };
if (o.replayReasoning) message.providerData = { kind: o.kind, data: { reasoning_content: reasoningText } };
yield {
type: 'done',
message,
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
};
}
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const base = normalizeCompatBase(o.baseUrl);
let res: unknown;
try {
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
} catch (err) {
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
}
throw err;
}
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
const found: DiscoveredModel[] = [];
for (const m of list) {
if (!isRecord(m)) continue;
const id = str(m.id) || str(m.name);
if (!id || !isChatModelId(id)) continue;
const model: DiscoveredModel = { id };
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
// OpenRouter-style metadata, when present.
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
if (ctxLen) model.contextWindow = ctxLen;
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
if (isRecord(m.pricing)) {
const pin = Number(m.pricing.prompt);
const pout = Number(m.pricing.completion);
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
const pcache = Number(m.pricing.input_cache_read);
if (m.pricing.input_cache_read !== undefined && Number.isFinite(pcache) && pcache >= 0) model.priceCachedIn = Math.round(pcache * 1e9) / 1e3;
}
const caps: Partial<ModelCapabilities> = {};
if (Array.isArray(m.supported_parameters)) {
caps.tools = m.supported_parameters.includes('tools');
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
}
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
caps.vision = m.architecture.input_modalities.includes('image');
}
if (Object.keys(caps).length > 0) model.capabilities = caps;
found.push(model);
}
return withCatalogDefaults(o.kind, found);
}
/** OpenAI-style `/embeddings` (OpenAI-compatible servers, Ollama's included). */
export async function compatEmbed(o: CompatOptions, req: EmbedRequest): Promise<EmbedResult> {
const body: Record<string, unknown> = { model: req.model, input: req.inputs, encoding_format: 'float' };
if (req.dimensions && /text-embedding-3/.test(req.model)) body.dimensions = req.dimensions;
const res = await requestJson(`${normalizeCompatBase(o.baseUrl)}/embeddings`, { ...errorContext(o), headers: authHeaders(o), body, signal: req.signal, firstByteTimeoutMs: 120_000, totalTimeoutMs: 300_000 });
return parseEmbeddings(res, req.inputs.length, o.label);
}
function options(cfg: ProviderConfig): CompatOptions {
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
}
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
return runTest(cfg.name || 'the server', async () => {
const o = options(cfg);
try {
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
} catch (err) {
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
// No model listing: prove the connection with a one-token completion instead.
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
try {
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
if (ev.type === 'done') break;
}
} finally {
clearTimeout(timer);
}
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
}
});
}
export const openaiCompatibleAdapter: ProviderAdapter = {
kind: 'openai_compatible',
meta: {
label: 'OpenAI-compatible',
defaultBaseUrl: null,
requiresApiKey: false,
requiresBaseUrl: true,
supportsDiscovery: true,
docsUrl: 'https://openrouter.ai/docs/quickstart',
keyHint: 'Optional for local servers',
},
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
test,
embed: (cfg, req) => compatEmbed(options(cfg), req),
};