All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
- Drag-to-reorder for class materials, folders, quiz questions and deck cards. Pointer-based so touch works, plus keyboard (arrows/Home/End) with screen-reader announcements. - Timed exams get a wall-clock deadline. The server refuses late answers, submits overdue exams with what was saved, and the countdown keeps running while the page is closed. - Shared quiz sessions keep a frozen copy of their questions and flag later owner edits. - Scanned PDFs are read in cached 8-page batches with real page numbers (pdf-lib), removing the 30 MB whole-file limit. Compatible providers now receive PDFs as file parts. - Semantic search: embeddings for OpenAI, Gemini, Ollama and compatible servers, background indexing into chunk_vectors (migration 2), hybrid keyword + vector ranking for search, the assistant and focused generation, and a Settings -> AI section. 181 tests pass locally and in the Docker test stage on Node 24. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
17 KiB
TypeScript
374 lines
17 KiB
TypeScript
/**
|
|
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
|
|
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
|
|
* Ollama adapter.
|
|
*
|
|
* These servers agree on the basics and disagree on everything else, so this
|
|
* adapter is deliberately forgiving:
|
|
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
|
|
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
|
|
* once when a server answers 400 about them;
|
|
* - usage may be missing, and may arrive on a chunk with no choices;
|
|
* - tool-call argument fragments may arrive in any chunking, with or without ids;
|
|
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
|
|
* - the last chunk may lack a `finish_reason`.
|
|
*/
|
|
import { type EmbedRequest, type EmbedResult,
|
|
AIError,
|
|
type ChatMessage,
|
|
type ChatRequest,
|
|
type ContentPart,
|
|
type DiscoveredModel,
|
|
type ModelCapabilities,
|
|
type ProviderAdapter,
|
|
type ProviderConfig,
|
|
type ProviderKind,
|
|
type StopReason,
|
|
type StreamEvent,
|
|
type TestResult,
|
|
} from '../types.ts';
|
|
import { parseEmbeddings, dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
|
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
|
|
|
type Json = Record<string, unknown>;
|
|
|
|
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
|
|
export function normalizeCompatBase(url: string): string {
|
|
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
|
|
try {
|
|
const u = new URL(base);
|
|
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
|
|
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
|
|
} catch {
|
|
/* validated by the caller */
|
|
}
|
|
return base;
|
|
}
|
|
|
|
export interface CompatOptions {
|
|
kind: ProviderKind;
|
|
label: string;
|
|
baseUrl: string;
|
|
apiKey: string | null;
|
|
headers?: Record<string, string>;
|
|
}
|
|
|
|
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
|
|
const blocks: Json[] = [];
|
|
for (const p of parts) {
|
|
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
|
else if (p.type === 'image') {
|
|
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
|
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
|
|
} else if (p.type === 'file') {
|
|
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
|
// Chat Completions "file" parts (OpenAI, OpenRouter and others) for models marked as reading PDFs.
|
|
else if (p.mime === 'application/pdf' && caps.pdf) blocks.push({ type: 'file', file: { filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) } });
|
|
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
|
}
|
|
}
|
|
// Plain strings are understood by every server; arrays only by multimodal ones.
|
|
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
|
|
return blocks;
|
|
}
|
|
|
|
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
|
|
const caps = req.capabilities ?? {};
|
|
const out: Json[] = [];
|
|
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
|
|
for (const msg of req.messages) {
|
|
if (msg.role === 'assistant') {
|
|
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
|
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
|
|
if (!text && calls.length === 0) continue;
|
|
const wire: Json = { role: 'assistant', content: text || null };
|
|
if (calls.length > 0) {
|
|
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
|
|
}
|
|
out.push(wire);
|
|
continue;
|
|
}
|
|
// Tool results must directly follow the assistant turn that requested them.
|
|
for (const p of msg.content) {
|
|
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
|
|
}
|
|
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
|
|
if (rest.length > 0) {
|
|
const content = userContent(rest, caps);
|
|
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
interface BodyFlags {
|
|
tools: boolean;
|
|
usage: boolean;
|
|
jsonMode: boolean;
|
|
temperature: boolean;
|
|
reasoning: boolean;
|
|
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
|
|
}
|
|
|
|
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
|
|
const caps = req.capabilities ?? {};
|
|
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
|
|
if (flags.usage) body.stream_options = { include_usage: true };
|
|
if (flags.tools && caps.tools !== false && req.tools?.length) {
|
|
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
|
|
const choice = req.toolChoice;
|
|
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
|
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
|
|
}
|
|
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
|
|
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
|
|
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
|
|
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
|
|
return body;
|
|
}
|
|
|
|
function mapFinish(reason: string): StopReason {
|
|
switch (reason) {
|
|
case 'stop':
|
|
case 'eos':
|
|
case 'end_turn':
|
|
return 'end';
|
|
case 'tool_calls':
|
|
case 'function_call':
|
|
return 'tool_use';
|
|
case 'length':
|
|
case 'max_tokens':
|
|
return 'max_tokens';
|
|
case 'content_filter':
|
|
return 'refusal';
|
|
default:
|
|
return 'other';
|
|
}
|
|
}
|
|
|
|
function authHeaders(o: CompatOptions): Record<string, string> {
|
|
const h: Record<string, string> = { ...o.headers };
|
|
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
|
|
return h;
|
|
}
|
|
|
|
function errorContext(o: CompatOptions): ErrorContext {
|
|
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
|
|
}
|
|
|
|
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
|
const ctx = errorContext(o);
|
|
const base = normalizeCompatBase(o.baseUrl);
|
|
const local = isLocalUrl(base);
|
|
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
|
|
|
|
let http: HttpResponse;
|
|
for (let attempt = 0; ; attempt += 1) {
|
|
http = await httpRequest(`${base}/chat/completions`, {
|
|
...ctx,
|
|
headers: authHeaders(o),
|
|
body: buildBody(req, flags),
|
|
signal: req.signal,
|
|
// A local server may spend minutes loading a model into memory before the first byte.
|
|
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
|
|
});
|
|
try {
|
|
await ensureOk(http);
|
|
break;
|
|
} catch (err) {
|
|
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
|
|
const detail = (err.providerMessage ?? '').toLowerCase();
|
|
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
|
|
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
|
|
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
|
|
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
|
|
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
|
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
|
|
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
|
|
// The model (or server) cannot do tool calling at all.
|
|
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
|
|
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
|
|
status: err.status,
|
|
providerMessage: err.providerMessage,
|
|
});
|
|
}
|
|
flags.tools = false;
|
|
} else throw err;
|
|
}
|
|
}
|
|
|
|
let text = '';
|
|
let finish = '';
|
|
let reasoningSeen = false;
|
|
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
|
const calls: { id: string; name: string; args: string }[] = [];
|
|
|
|
for await (const { data } of sseJson(http)) {
|
|
if (!isRecord(data)) continue;
|
|
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
|
if (isRecord(data.usage)) {
|
|
const u = data.usage;
|
|
usage.input = num(u.prompt_tokens) || usage.input;
|
|
usage.output = num(u.completion_tokens) || usage.output;
|
|
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
|
|
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
|
|
}
|
|
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
|
|
if (!isRecord(choice)) continue;
|
|
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
|
|
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
|
|
if (!delta) continue;
|
|
|
|
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
|
|
if (reasoning) {
|
|
reasoningSeen = true;
|
|
yield { type: 'reasoning', delta: reasoning };
|
|
}
|
|
const piece = typeof delta.content === 'string' ? delta.content : '';
|
|
if (piece) {
|
|
text += piece;
|
|
yield { type: 'text', delta: piece };
|
|
}
|
|
if (Array.isArray(delta.tool_calls)) {
|
|
for (const raw of delta.tool_calls) {
|
|
if (!isRecord(raw)) continue;
|
|
const fn = isRecord(raw.function) ? raw.function : {};
|
|
// `index` identifies the call; servers that omit it send one whole call per entry.
|
|
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
|
|
let slot = index >= 0 ? calls[index] : undefined;
|
|
if (!slot) {
|
|
slot = { id: '', name: '', args: '' };
|
|
if (index >= 0) calls[index] = slot;
|
|
else calls.push(slot);
|
|
}
|
|
if (str(raw.id)) slot.id = str(raw.id);
|
|
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
|
|
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
|
|
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
|
|
}
|
|
}
|
|
}
|
|
|
|
const stop = finish ? mapFinish(finish) : 'end';
|
|
const content: ContentPart[] = [];
|
|
if (text) content.push({ type: 'text', text });
|
|
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
|
for (const c of calls) {
|
|
if (!c || !c.name) continue;
|
|
let input = parseArgsStrict(c.args);
|
|
if (!input) {
|
|
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
|
|
input = parseToolArguments(c.args, o.label, c.name);
|
|
}
|
|
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
|
|
}
|
|
for (const call of done) {
|
|
content.push({ type: 'tool_call', ...call });
|
|
yield { type: 'tool_call', call };
|
|
}
|
|
if (content.length === 0 && !reasoningSeen && !finish) {
|
|
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
|
|
}
|
|
const message: ChatMessage = { role: 'assistant', content };
|
|
yield {
|
|
type: 'done',
|
|
message,
|
|
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
|
|
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
|
};
|
|
}
|
|
|
|
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
|
const base = normalizeCompatBase(o.baseUrl);
|
|
let res: unknown;
|
|
try {
|
|
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
|
} catch (err) {
|
|
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
|
|
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
|
|
}
|
|
throw err;
|
|
}
|
|
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
|
|
const found: DiscoveredModel[] = [];
|
|
for (const m of list) {
|
|
if (!isRecord(m)) continue;
|
|
const id = str(m.id) || str(m.name);
|
|
if (!id || !isChatModelId(id)) continue;
|
|
const model: DiscoveredModel = { id };
|
|
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
|
|
// OpenRouter-style metadata, when present.
|
|
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
|
|
if (ctxLen) model.contextWindow = ctxLen;
|
|
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
|
|
if (isRecord(m.pricing)) {
|
|
const pin = Number(m.pricing.prompt);
|
|
const pout = Number(m.pricing.completion);
|
|
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
|
|
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
|
|
}
|
|
const caps: Partial<ModelCapabilities> = {};
|
|
if (Array.isArray(m.supported_parameters)) {
|
|
caps.tools = m.supported_parameters.includes('tools');
|
|
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
|
|
}
|
|
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
|
|
caps.vision = m.architecture.input_modalities.includes('image');
|
|
}
|
|
if (Object.keys(caps).length > 0) model.capabilities = caps;
|
|
found.push(model);
|
|
}
|
|
return withCatalogDefaults(o.kind, found);
|
|
}
|
|
|
|
/** OpenAI-style `/embeddings`, shared with Ollama's compatible endpoint. */
|
|
export async function compatEmbed(o: CompatOptions, req: EmbedRequest): Promise<EmbedResult> {
|
|
const body: Record<string, unknown> = { model: req.model, input: req.inputs, encoding_format: 'float' };
|
|
if (req.dimensions && /text-embedding-3/.test(req.model)) body.dimensions = req.dimensions;
|
|
const res = await requestJson(`${normalizeCompatBase(o.baseUrl)}/embeddings`, { ...errorContext(o), headers: authHeaders(o), body, signal: req.signal, firstByteTimeoutMs: 120_000, totalTimeoutMs: 300_000 });
|
|
return parseEmbeddings(res, req.inputs.length, o.label);
|
|
}
|
|
|
|
function options(cfg: ProviderConfig): CompatOptions {
|
|
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
|
|
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
|
|
}
|
|
|
|
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
|
|
return runTest(cfg.name || 'the server', async () => {
|
|
const o = options(cfg);
|
|
try {
|
|
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
|
|
} catch (err) {
|
|
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
|
|
// No model listing: prove the connection with a one-token completion instead.
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
|
|
try {
|
|
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
|
|
if (ev.type === 'done') break;
|
|
}
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
|
|
}
|
|
});
|
|
}
|
|
|
|
export const openaiCompatibleAdapter: ProviderAdapter = {
|
|
kind: 'openai_compatible',
|
|
meta: {
|
|
label: 'OpenAI-compatible',
|
|
defaultBaseUrl: null,
|
|
requiresApiKey: false,
|
|
requiresBaseUrl: true,
|
|
supportsDiscovery: true,
|
|
docsUrl: 'https://openrouter.ai/docs/quickstart',
|
|
keyHint: 'Optional for local servers',
|
|
},
|
|
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
|
|
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
|
|
test,
|
|
embed: (cfg, req) => compatEmbed(options(cfg), req),
|
|
};
|