All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 25s
- Output limits leave room for reasoning (answer plus 8k-64k by effort, up to the model maximum); a call cut off before answering is retried with double the limit. At "max" effort, batches were being cut off at 16k tokens. - Quizzes are planned first (one distinct angle per question), then written in waves of four batches; med math runs alongside the first wave. - Replacements for removed questions are planned and written in parallel. - The Python check and the second review run at the same time each round. Measured live (DeepSeek flash, max reasoning, 50 questions): 7 min 21 s with no duplicates, against 20 min and 29 replaced duplicates without planning. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
776 lines
35 KiB
TypeScript
776 lines
35 KiB
TypeScript
/**
|
|
* Provider + model configuration, task routing, usage accounting and the two call
|
|
* primitives everything else builds on: `runChat` (one streamed model turn) and
|
|
* `generateStructured` (schema-constrained JSON through a forced tool call).
|
|
*/
|
|
import { jsonrepair } from 'jsonrepair';
|
|
import { z } from 'zod';
|
|
import { decryptSecret, encryptSecret, maskSecret } from '../crypto.ts';
|
|
import type { DB } from '../db.ts';
|
|
import { HttpError, notFound, type AppContext } from '../http.ts';
|
|
import { getSetting, setSetting } from '../settings.ts';
|
|
import { newId, now, parseJson } from '../util.ts';
|
|
import { clipOutput, mcpActivity, type ToolActivity } from './activity.ts';
|
|
import { loadMcpTools, mcpPromptNote, type McpArea } from './mcp.ts';
|
|
import { cachedInputPrice, cacheWriteRatio, getAdapter, lookupModelDefaults, priceMultiplier } from './providers/index.ts';
|
|
import {
|
|
AIError,
|
|
DEFAULT_CAPABILITIES,
|
|
PROVIDER_KINDS,
|
|
REASONING_EFFORTS,
|
|
type ChatMessage,
|
|
type ContentPart,
|
|
type ChatRequest,
|
|
type DiscoveredModel,
|
|
type ModelCapabilities,
|
|
type ProviderConfig,
|
|
type ProviderKind,
|
|
type ReasoningEffort,
|
|
type StopReason,
|
|
type StreamEvent,
|
|
type ToolDef,
|
|
type Usage,
|
|
} from './types.ts';
|
|
|
|
/* ------------------------------ rows + DTOs ------------------------------ */
|
|
|
|
interface ProviderRow {
|
|
id: string;
|
|
name: string;
|
|
kind: ProviderKind;
|
|
base_url: string | null;
|
|
api_key_enc: string | null;
|
|
headers_enc: string | null;
|
|
enabled: number;
|
|
last_test_at: number | null;
|
|
last_test_ok: number | null;
|
|
last_test_message: string;
|
|
created_at: number;
|
|
updated_at: number;
|
|
}
|
|
|
|
interface ModelRow {
|
|
id: string;
|
|
provider_id: string;
|
|
model: string;
|
|
label: string;
|
|
source: string;
|
|
enabled: number;
|
|
capabilities: string;
|
|
context_window: number | null;
|
|
max_output: number | null;
|
|
price_in: number | null;
|
|
price_out: number | null;
|
|
price_cached_in: number | null;
|
|
}
|
|
|
|
export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const;
|
|
export type TaskName = (typeof TASKS)[number];
|
|
|
|
export const TASK_INFO: Record<TaskName, { label: string; description: string }> = {
|
|
chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' },
|
|
generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' },
|
|
vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' },
|
|
quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' },
|
|
};
|
|
|
|
export interface TaskConfig {
|
|
modelId: string | null;
|
|
temperature: number | null;
|
|
reasoningEffort: ReasoningEffort | null;
|
|
maxOutputTokens: number | null;
|
|
}
|
|
export type TaskSettings = Record<TaskName | 'default', TaskConfig>;
|
|
|
|
const EMPTY_TASK: TaskConfig = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
|
|
|
|
export const taskConfigSchema = z.object({
|
|
modelId: z.string().nullable(),
|
|
temperature: z.number().min(0).max(2).nullable(),
|
|
reasoningEffort: z.enum(REASONING_EFFORTS).nullable(),
|
|
maxOutputTokens: z.number().int().min(256).max(200000).nullable(),
|
|
});
|
|
|
|
function mapProvider(app: AppContext, row: ProviderRow) {
|
|
const key = decryptSecret(row.api_key_enc, app.config.secretKey);
|
|
const headers = parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {});
|
|
return {
|
|
id: row.id,
|
|
name: row.name,
|
|
kind: row.kind,
|
|
baseUrl: row.base_url,
|
|
hasKey: Boolean(row.api_key_enc),
|
|
/** Only ever a masked preview; the key itself never leaves the server. */
|
|
keyPreview: key ? maskSecret(key) : row.api_key_enc ? 'unreadable (re-enter key)' : '',
|
|
headerNames: Object.keys(headers),
|
|
enabled: Boolean(row.enabled),
|
|
lastTestAt: row.last_test_at,
|
|
lastTestOk: row.last_test_ok === null ? null : Boolean(row.last_test_ok),
|
|
lastTestMessage: row.last_test_message,
|
|
createdAt: row.created_at,
|
|
};
|
|
}
|
|
|
|
function mapModel(row: ModelRow & { provider_name?: string; provider_kind?: string; provider_enabled?: number }) {
|
|
return {
|
|
id: row.id,
|
|
providerId: row.provider_id,
|
|
providerName: row.provider_name ?? '',
|
|
providerKind: (row.provider_kind ?? '') as ProviderKind,
|
|
model: row.model,
|
|
label: row.label || row.model,
|
|
source: row.source,
|
|
enabled: Boolean(row.enabled),
|
|
available: Boolean(row.enabled) && row.provider_enabled !== 0,
|
|
capabilities: { ...DEFAULT_CAPABILITIES, ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}) },
|
|
contextWindow: row.context_window,
|
|
maxOutput: row.max_output,
|
|
priceIn: row.price_in,
|
|
priceOut: row.price_out,
|
|
/** Null = not set; costs then use the provider's usual cache discount. */
|
|
priceCachedIn: row.price_cached_in,
|
|
};
|
|
}
|
|
export type ModelDto = ReturnType<typeof mapModel>;
|
|
|
|
const MODEL_SELECT = `SELECT m.*, p.name AS provider_name, p.kind AS provider_kind, p.enabled AS provider_enabled FROM ai_models m JOIN ai_providers p ON p.id = m.provider_id`;
|
|
|
|
/* ------------------------------ providers ------------------------------ */
|
|
|
|
export const providerInputSchema = z.object({
|
|
name: z.string().trim().min(1).max(80),
|
|
kind: z.enum(PROVIDER_KINDS as [ProviderKind, ...ProviderKind[]]),
|
|
baseUrl: z
|
|
.string()
|
|
.trim()
|
|
.max(500)
|
|
.nullish()
|
|
.transform((v) => v || null)
|
|
.refine((v) => v === null || /^https?:\/\//i.test(v), 'URL must start with http:// or https://'),
|
|
/** undefined = keep the stored key, '' or null = remove it. */
|
|
apiKey: z.string().max(2000).nullish(),
|
|
headers: z.record(z.string().max(100), z.string().max(2000)).optional(),
|
|
enabled: z.boolean().optional(),
|
|
});
|
|
|
|
function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig {
|
|
return {
|
|
id: row.id,
|
|
kind: row.kind,
|
|
name: row.name,
|
|
baseUrl: row.base_url,
|
|
apiKey: decryptSecret(row.api_key_enc, app.config.secretKey),
|
|
headers: parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {}),
|
|
};
|
|
}
|
|
|
|
/** Decrypted connection details for one enabled provider (server-side use only, never sent to clients). */
|
|
export function providerConfigById(app: AppContext, id: string): { cfg: ProviderConfig; kind: ProviderKind; name: string } {
|
|
const row = providerRow(app.db, id);
|
|
if (!row.enabled) throw new AIError('not_configured', `The provider "${row.name}" is turned off. Turn it on in Settings.`);
|
|
return { cfg: providerConfig(app, row), kind: row.kind, name: row.name };
|
|
}
|
|
|
|
function providerRow(db: DB, id: string): ProviderRow {
|
|
const row = db.get<ProviderRow>('SELECT * FROM ai_providers WHERE id = ?', id);
|
|
if (!row) throw notFound('Provider');
|
|
return row;
|
|
}
|
|
|
|
export function listProviders(app: AppContext) {
|
|
return app.db.all<ProviderRow>('SELECT * FROM ai_providers ORDER BY created_at').map((r) => mapProvider(app, r));
|
|
}
|
|
|
|
export function saveProvider(app: AppContext, id: string | null, input: z.input<typeof providerInputSchema>) {
|
|
const data = providerInputSchema.parse(input);
|
|
const meta = getAdapter(data.kind).meta;
|
|
if (meta.requiresBaseUrl && !data.baseUrl) throw new HttpError(422, 'validation', 'This provider type needs a server URL');
|
|
const at = now();
|
|
const key = app.config.secretKey;
|
|
if (id) {
|
|
const row = providerRow(app.db, id);
|
|
const apiKeyEnc = data.apiKey === undefined ? row.api_key_enc : data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null;
|
|
const headersEnc = data.headers === undefined ? row.headers_enc : Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null;
|
|
app.db.run(
|
|
'UPDATE ai_providers SET name = ?, kind = ?, base_url = ?, api_key_enc = ?, headers_enc = ?, enabled = ?, updated_at = ? WHERE id = ?',
|
|
data.name,
|
|
data.kind,
|
|
data.baseUrl,
|
|
apiKeyEnc,
|
|
headersEnc,
|
|
data.enabled === undefined ? row.enabled : data.enabled ? 1 : 0,
|
|
at,
|
|
id,
|
|
);
|
|
return mapProvider(app, providerRow(app.db, id));
|
|
}
|
|
const newProviderId = newId();
|
|
app.db.run(
|
|
'INSERT INTO ai_providers (id, name, kind, base_url, api_key_enc, headers_enc, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
|
newProviderId,
|
|
data.name,
|
|
data.kind,
|
|
data.baseUrl,
|
|
data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null,
|
|
data.headers && Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null,
|
|
data.enabled === false ? 0 : 1,
|
|
at,
|
|
at,
|
|
);
|
|
return mapProvider(app, providerRow(app.db, newProviderId));
|
|
}
|
|
|
|
export function deleteProvider(app: AppContext, id: string): void {
|
|
providerRow(app.db, id);
|
|
const modelIds = new Set(app.db.all<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ?', id).map((m) => m.id));
|
|
app.db.run('DELETE FROM ai_providers WHERE id = ?', id);
|
|
const tasks = getTaskSettings(app.db);
|
|
let changed = false;
|
|
for (const name of Object.keys(tasks) as (keyof TaskSettings)[]) {
|
|
if (tasks[name].modelId && modelIds.has(tasks[name].modelId!)) {
|
|
tasks[name].modelId = null;
|
|
changed = true;
|
|
}
|
|
}
|
|
if (changed) setSetting(app.db, 'ai.tasks', tasks);
|
|
}
|
|
|
|
export async function testProvider(app: AppContext, id: string) {
|
|
const row = providerRow(app.db, id);
|
|
const anyModel = app.db.get<{ model: string }>('SELECT model FROM ai_models WHERE provider_id = ? ORDER BY enabled DESC LIMIT 1', id)?.model;
|
|
const result = await getAdapter(row.kind).test(providerConfig(app, row), anyModel);
|
|
app.db.run('UPDATE ai_providers SET last_test_at = ?, last_test_ok = ?, last_test_message = ? WHERE id = ?', now(), result.ok ? 1 : 0, result.message.slice(0, 500), id);
|
|
return result;
|
|
}
|
|
|
|
/** Looks up the provider's model list and merges it into ai_models without disturbing user edits. */
|
|
export async function discoverModels(app: AppContext, id: string): Promise<{ added: number; total: number }> {
|
|
const row = providerRow(app.db, id);
|
|
const found = await getAdapter(row.kind).listModels(providerConfig(app, row));
|
|
const existing = new Map(app.db.all<ModelRow>('SELECT * FROM ai_models WHERE provider_id = ?', id).map((m) => [m.model, m]));
|
|
const at = now();
|
|
let added = 0;
|
|
app.db.tx(() => {
|
|
for (const m of found) {
|
|
const current = existing.get(m.id);
|
|
if (current) {
|
|
// Refresh facts the provider reports, but never overwrite prices or flags the user may have edited.
|
|
app.db.run(
|
|
'UPDATE ai_models SET context_window = COALESCE(?, context_window), max_output = COALESCE(?, max_output), price_in = COALESCE(price_in, ?), price_out = COALESCE(price_out, ?), price_cached_in = COALESCE(price_cached_in, ?), updated_at = ? WHERE id = ?',
|
|
m.contextWindow ?? null,
|
|
m.maxOutput ?? null,
|
|
m.priceIn ?? null,
|
|
m.priceOut ?? null,
|
|
m.priceCachedIn ?? null,
|
|
at,
|
|
current.id,
|
|
);
|
|
continue;
|
|
}
|
|
insertModel(app.db, id, m, 'discovered', false);
|
|
added += 1;
|
|
}
|
|
});
|
|
return { added, total: found.length };
|
|
}
|
|
|
|
function insertModel(db: DB, providerId: string, m: DiscoveredModel, source: string, enabled: boolean): string {
|
|
const id = newId();
|
|
const at = now();
|
|
db.run(
|
|
`INSERT INTO ai_models (id, provider_id, model, label, source, enabled, capabilities, context_window, max_output, price_in, price_out, price_cached_in, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
id,
|
|
providerId,
|
|
m.id,
|
|
m.label ?? '',
|
|
source,
|
|
enabled ? 1 : 0,
|
|
JSON.stringify(m.capabilities ?? {}),
|
|
m.contextWindow ?? null,
|
|
m.maxOutput ?? null,
|
|
m.priceIn ?? null,
|
|
m.priceOut ?? null,
|
|
m.priceCachedIn ?? null,
|
|
at,
|
|
at,
|
|
);
|
|
return id;
|
|
}
|
|
|
|
/* ------------------------------ models ------------------------------ */
|
|
|
|
export function listModels(db: DB): ModelDto[] {
|
|
return db.all<ModelRow>(`${MODEL_SELECT} ORDER BY p.created_at, m.enabled DESC, m.model`).map(mapModel);
|
|
}
|
|
|
|
export function getModel(db: DB, id: string): ModelDto {
|
|
const row = db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ?`, id);
|
|
if (!row) throw notFound('Model');
|
|
return mapModel(row);
|
|
}
|
|
|
|
export const manualModelSchema = z.object({
|
|
providerId: z.string(),
|
|
model: z.string().trim().min(1).max(200),
|
|
label: z.string().trim().max(120).optional(),
|
|
});
|
|
|
|
export function addManualModel(app: AppContext, input: z.input<typeof manualModelSchema>): ModelDto {
|
|
const data = manualModelSchema.parse(input);
|
|
const provider = providerRow(app.db, data.providerId);
|
|
const existing = app.db.get<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ? AND model = ?', provider.id, data.model);
|
|
if (existing) {
|
|
app.db.run('UPDATE ai_models SET enabled = 1, updated_at = ? WHERE id = ?', now(), existing.id);
|
|
return getModel(app.db, existing.id);
|
|
}
|
|
const defaults = lookupModelDefaults(provider.kind, data.model);
|
|
const id = insertModel(app.db, provider.id, { ...defaults, id: data.model, label: data.label || defaults.label }, 'manual', true);
|
|
return getModel(app.db, id);
|
|
}
|
|
|
|
export const modelPatchSchema = z.object({
|
|
enabled: z.boolean().optional(),
|
|
label: z.string().trim().max(120).optional(),
|
|
capabilities: z
|
|
.object({ tools: z.boolean(), vision: z.boolean(), pdf: z.boolean(), reasoning: z.boolean(), temperature: z.boolean() })
|
|
.partial()
|
|
.optional(),
|
|
contextWindow: z.number().int().min(256).max(100_000_000).nullable().optional(),
|
|
maxOutput: z.number().int().min(16).max(10_000_000).nullable().optional(),
|
|
priceIn: z.number().min(0).max(100000).nullable().optional(),
|
|
priceOut: z.number().min(0).max(100000).nullable().optional(),
|
|
priceCachedIn: z.number().min(0).max(100000).nullable().optional(),
|
|
});
|
|
|
|
export function patchModel(db: DB, id: string, input: z.input<typeof modelPatchSchema>): ModelDto {
|
|
const patch = modelPatchSchema.parse(input);
|
|
const row = db.get<ModelRow>('SELECT * FROM ai_models WHERE id = ?', id);
|
|
if (!row) throw notFound('Model');
|
|
const caps = { ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}), ...patch.capabilities };
|
|
db.run(
|
|
'UPDATE ai_models SET enabled = ?, label = ?, capabilities = ?, context_window = ?, max_output = ?, price_in = ?, price_out = ?, price_cached_in = ?, updated_at = ? WHERE id = ?',
|
|
patch.enabled === undefined ? row.enabled : patch.enabled ? 1 : 0,
|
|
patch.label ?? row.label,
|
|
JSON.stringify(caps),
|
|
patch.contextWindow === undefined ? row.context_window : patch.contextWindow,
|
|
patch.maxOutput === undefined ? row.max_output : patch.maxOutput,
|
|
patch.priceIn === undefined ? row.price_in : patch.priceIn,
|
|
patch.priceOut === undefined ? row.price_out : patch.priceOut,
|
|
patch.priceCachedIn === undefined ? row.price_cached_in : patch.priceCachedIn,
|
|
now(),
|
|
id,
|
|
);
|
|
return getModel(db, id);
|
|
}
|
|
|
|
export function deleteModel(db: DB, id: string): void {
|
|
const { changes } = db.run('DELETE FROM ai_models WHERE id = ?', id);
|
|
if (!changes) throw notFound('Model');
|
|
}
|
|
|
|
/* ------------------------------ task routing ------------------------------ */
|
|
|
|
export function getTaskSettings(db: DB): TaskSettings {
|
|
const stored = getSetting<Partial<TaskSettings>>(db, 'ai.tasks', {});
|
|
const out = { default: { ...EMPTY_TASK, ...stored.default } } as TaskSettings;
|
|
for (const t of TASKS) out[t] = { ...EMPTY_TASK, ...stored[t] };
|
|
return out;
|
|
}
|
|
|
|
export function saveTaskSettings(db: DB, input: unknown): TaskSettings {
|
|
const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial();
|
|
const patch = schema.parse(input);
|
|
const next = { ...getTaskSettings(db), ...patch } as TaskSettings;
|
|
setSetting(db, 'ai.tasks', next);
|
|
return next;
|
|
}
|
|
|
|
export interface ResolvedModel {
|
|
provider: ProviderConfig;
|
|
model: ModelDto;
|
|
params: { temperature?: number; reasoningEffort?: ReasoningEffort; maxOutputTokens?: number };
|
|
}
|
|
|
|
/**
|
|
* Picks the model for a task: explicit override, then the task's own model, then the
|
|
* default model, then (so a fresh install works after adding one model) any enabled model.
|
|
*/
|
|
export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel {
|
|
const tasks = getTaskSettings(app.db);
|
|
const cfg = tasks[task];
|
|
const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x));
|
|
let model: ModelDto | undefined;
|
|
for (const id of candidates) {
|
|
const row = app.db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id);
|
|
if (row) {
|
|
model = mapModel(row);
|
|
break;
|
|
}
|
|
}
|
|
if (!model) {
|
|
const needVision = task === 'vision';
|
|
const rows = app.db.all<ModelRow>(`${MODEL_SELECT} WHERE m.enabled = 1 AND p.enabled = 1 ORDER BY p.created_at, m.created_at`).map(mapModel);
|
|
model = needVision ? rows.find((m) => m.capabilities.vision) : rows[0];
|
|
}
|
|
if (!model) {
|
|
throw new AIError(
|
|
'not_configured',
|
|
task === 'vision' ? 'No vision-capable AI model is set up. Add one in Settings, then choose it for "Vision and OCR".' : 'No AI model is set up yet. Add a provider and enable a model in Settings.',
|
|
);
|
|
}
|
|
const provider = providerConfig(app, providerRow(app.db, model.providerId));
|
|
const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK;
|
|
const fallback = tasks.default;
|
|
const params: ResolvedModel['params'] = {};
|
|
const temperature = own.temperature ?? fallback.temperature;
|
|
const effort = own.reasoningEffort ?? fallback.reasoningEffort;
|
|
const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens;
|
|
if (temperature !== null && model.capabilities.temperature) params.temperature = temperature;
|
|
if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort;
|
|
if (maxOut !== null) params.maxOutputTokens = maxOut;
|
|
return { provider, model, params };
|
|
}
|
|
|
|
/* ------------------------------ usage ------------------------------ */
|
|
|
|
/**
|
|
* Estimated USD cost of one call. Input tokens are split into fresh, cache-hit and
|
|
* cache-write tokens, each at its own rate: the model's cached price when set, otherwise
|
|
* the provider's published cache discount; Claude's cache writes cost 1.25x. Providers
|
|
* with time-of-day pricing (DeepSeek off-peak) are adjusted for when the call was made.
|
|
*/
|
|
export function estimateCost(
|
|
model: Pick<ModelDto, 'priceIn' | 'priceOut' | 'model'> & { priceCachedIn?: number | null; providerKind?: ProviderKind | '' },
|
|
usage: Usage,
|
|
at: number = Date.now(),
|
|
): number | null {
|
|
if (model.priceIn === null && model.priceOut === null) return null;
|
|
const kind = (model.providerKind || 'openai_compatible') as ProviderKind;
|
|
const priceIn = model.priceIn ?? 0;
|
|
const cachedRate = cachedInputPrice(kind, model.model, model.priceIn, model.priceCachedIn ?? null);
|
|
const cached = Math.min(usage.inputTokens, usage.cacheReadTokens ?? 0);
|
|
const written = Math.min(usage.inputTokens - cached, usage.cacheWriteTokens ?? 0);
|
|
const fresh = Math.max(0, usage.inputTokens - cached - written);
|
|
const input = fresh * priceIn + cached * cachedRate + written * priceIn * cacheWriteRatio(kind);
|
|
return ((input + usage.outputTokens * (model.priceOut ?? 0)) * priceMultiplier(kind, at)) / 1_000_000;
|
|
}
|
|
|
|
function recordUsage(app: AppContext, r: ResolvedModel, task: string, usage: Usage | null, started: number, error?: string): void {
|
|
try {
|
|
app.db.run(
|
|
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, cost, ok, error, duration_ms)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
now(),
|
|
r.provider.id,
|
|
r.provider.name,
|
|
r.model.model,
|
|
task,
|
|
usage?.inputTokens ?? 0,
|
|
usage?.outputTokens ?? 0,
|
|
usage?.cacheReadTokens ?? 0,
|
|
usage?.cacheWriteTokens ?? 0,
|
|
usage ? estimateCost(r.model, usage) : null,
|
|
error ? 0 : 1,
|
|
(error ?? '').slice(0, 300),
|
|
now() - started,
|
|
);
|
|
} catch {
|
|
/* accounting must never break a request */
|
|
}
|
|
}
|
|
|
|
/* ------------------------------ call primitives ------------------------------ */
|
|
|
|
export interface ChatOptions {
|
|
task: TaskName;
|
|
/** Label stored with usage rows ("assistant", "quiz generation"...). Defaults to the task. */
|
|
purpose?: string;
|
|
modelId?: string | null;
|
|
system?: string;
|
|
messages: ChatMessage[];
|
|
tools?: ToolDef[];
|
|
toolChoice?: ChatRequest['toolChoice'];
|
|
jsonMode?: boolean;
|
|
temperature?: number;
|
|
reasoningEffort?: ReasoningEffort;
|
|
maxOutputTokens?: number;
|
|
signal?: AbortSignal;
|
|
onEvent?: (event: StreamEvent) => void;
|
|
}
|
|
|
|
export interface ChatResult {
|
|
message: ChatMessage;
|
|
text: string;
|
|
stopReason: StopReason;
|
|
usage: Usage;
|
|
model: ModelDto;
|
|
}
|
|
|
|
const sleep = (ms: number, signal?: AbortSignal) =>
|
|
new Promise<void>((resolve, reject) => {
|
|
const t = setTimeout(resolve, ms);
|
|
signal?.addEventListener('abort', () => {
|
|
clearTimeout(t);
|
|
reject(new AIError('aborted', 'The request was cancelled.'));
|
|
});
|
|
});
|
|
|
|
/** Runs one model turn. Retries transient failures, but only while nothing has been streamed yet. */
|
|
export async function runChat(app: AppContext, opts: ChatOptions): Promise<ChatResult> {
|
|
const resolved = resolveModel(app, opts.task, opts.modelId);
|
|
const caps = resolved.model.capabilities;
|
|
const req: ChatRequest = {
|
|
model: resolved.model.model,
|
|
system: opts.system,
|
|
messages: opts.messages,
|
|
tools: opts.tools,
|
|
toolChoice: opts.toolChoice,
|
|
jsonMode: opts.jsonMode,
|
|
capabilities: caps,
|
|
signal: opts.signal,
|
|
temperature: opts.temperature ?? resolved.params.temperature,
|
|
reasoningEffort: opts.reasoningEffort ?? resolved.params.reasoningEffort,
|
|
maxOutputTokens: opts.maxOutputTokens ?? resolved.params.maxOutputTokens ?? undefined,
|
|
};
|
|
if (req.maxOutputTokens && resolved.model.maxOutput) req.maxOutputTokens = Math.min(req.maxOutputTokens, resolved.model.maxOutput);
|
|
if (opts.tools?.length && !caps.tools && (opts.toolChoice === 'required' || typeof opts.toolChoice === 'object')) {
|
|
throw new AIError('unsupported', `${resolved.model.label} is marked as not supporting tools. Choose another model for this task.`);
|
|
}
|
|
const adapter = getAdapter(resolved.provider.kind);
|
|
const purpose = opts.purpose ?? opts.task;
|
|
|
|
for (let attempt = 0; ; attempt += 1) {
|
|
const started = now();
|
|
let streamed = false;
|
|
try {
|
|
for await (const event of adapter.stream(resolved.provider, req)) {
|
|
if (event.type === 'done') {
|
|
recordUsage(app, resolved, purpose, event.usage, started);
|
|
const text = event.message.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
|
return { message: event.message, text, stopReason: event.stopReason, usage: event.usage, model: resolved.model };
|
|
}
|
|
streamed = true;
|
|
opts.onEvent?.(event);
|
|
}
|
|
throw new AIError('invalid_response', `${resolved.provider.name} ended the response unexpectedly.`, { retryable: true });
|
|
} catch (raw) {
|
|
const err = raw instanceof AIError ? raw : new AIError('server', 'The AI request failed unexpectedly.', { cause: raw });
|
|
if (!(raw instanceof AIError)) app.log('error', 'Unexpected AI adapter error', raw);
|
|
recordUsage(app, resolved, purpose, null, started, `${err.code}: ${err.message}`);
|
|
if (!err.retryable || streamed || attempt >= 2 || opts.signal?.aborted) throw err;
|
|
await sleep(1500 * (attempt + 1) ** 2, opts.signal);
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Finds the first JSON value in model text (handles code fences and stray prose). */
|
|
export function parseLooseJson(text: string): unknown {
|
|
let t = text.trim();
|
|
const fence = /```(?:json)?\s*\n?([\s\S]*?)```/i.exec(t);
|
|
if (fence) t = fence[1].trim();
|
|
const start = t.search(/[[{]/);
|
|
if (start < 0) throw new AIError('invalid_response', 'The model did not return structured data.');
|
|
t = t.slice(start);
|
|
try {
|
|
return JSON.parse(t);
|
|
} catch {
|
|
try {
|
|
return JSON.parse(jsonrepair(t));
|
|
} catch {
|
|
throw new AIError('invalid_response', 'The model returned data that could not be read.', { retryable: true });
|
|
}
|
|
}
|
|
}
|
|
|
|
const MCP_ROUNDS = 4;
|
|
|
|
/**
|
|
* runChat with the MCP tools allowed in `area`. The model may call them for up to MCP_ROUNDS
|
|
* rounds; the result is the first reply with no MCP calls (or any call to the caller's own
|
|
* tools), so callers handle it exactly like a plain runChat result.
|
|
*/
|
|
export async function runChatWithMcp(app: AppContext, opts: ChatOptions, area: McpArea, hooks: { onTool?: (activity: ToolActivity) => void; exclude?: string[] } = {}): Promise<ChatResult> {
|
|
const resolved = resolveModel(app, opts.task, opts.modelId);
|
|
if (!resolved.model.capabilities.tools) return runChat(app, opts);
|
|
const mcp = await loadMcpTools(app, area, opts.signal, { exclude: hooks.exclude });
|
|
if (!mcp.tools.length) return runChat(app, opts);
|
|
const own = new Set((opts.tools ?? []).map((t) => t.name));
|
|
const forced = opts.toolChoice === 'required' || typeof opts.toolChoice === 'object';
|
|
const system = `${opts.system ?? ''}${mcpPromptNote(mcp)}${forced ? `\n\nYou may call the external tools first if they help; finish by calling ${typeof opts.toolChoice === 'object' ? opts.toolChoice.name : 'the required tool'}.` : ''}`;
|
|
const tools = [...(opts.tools ?? []), ...mcp.tools.map((t) => t.def)];
|
|
const messages = [...opts.messages];
|
|
for (let round = 0; ; round += 1) {
|
|
const last = round >= MCP_ROUNDS;
|
|
// While MCP rounds remain a forced choice becomes "some tool"; on the last round the original choice returns.
|
|
const toolChoice = last ? (forced ? opts.toolChoice : 'none') : forced ? 'required' : (opts.toolChoice ?? 'auto');
|
|
const res = await runChat(app, { ...opts, system, messages, tools, toolChoice });
|
|
const calls = res.message.content.filter((p): p is Extract<ContentPart, { type: 'tool_call' }> => p.type === 'tool_call');
|
|
const external = calls.filter((c) => !own.has(c.name) && mcp.tools.some((t) => t.def.name === c.name));
|
|
if (last || external.length === 0 || calls.some((c) => own.has(c.name))) return res;
|
|
const results: ContentPart[] = [];
|
|
for (const call of calls) {
|
|
const binding = mcp.tools.find((t) => t.def.name === call.name);
|
|
let output: string;
|
|
let isError = false;
|
|
const activity: ToolActivity = binding ? mcpActivity(binding, call.id, call.input) : { id: call.id, name: call.name, label: call.name, status: 'running', kind: 'mcp' };
|
|
try {
|
|
if (!binding) throw new Error(`Unknown tool "${call.name}"`);
|
|
hooks.onTool?.({ ...activity });
|
|
const result = await binding.call(call.input, opts.signal);
|
|
output = result.text;
|
|
activity.status = 'done';
|
|
activity.sources = result.sources;
|
|
activity.output = clipOutput(result.text);
|
|
} catch (err) {
|
|
if (opts.signal?.aborted) throw new AIError('aborted', 'Cancelled.');
|
|
isError = true;
|
|
const message = err instanceof Error ? err.message : 'Tool failed';
|
|
output = JSON.stringify({ error: message });
|
|
activity.status = 'error';
|
|
activity.summary = message.slice(0, 200);
|
|
}
|
|
hooks.onTool?.({ ...activity });
|
|
if (output.length > 24_000) output = `${output.slice(0, 24_000)}… [truncated]`;
|
|
results.push({ type: 'tool_result', toolCallId: call.id, name: call.name, output, isError });
|
|
}
|
|
messages.push(res.message, { role: 'user', content: results });
|
|
}
|
|
}
|
|
|
|
/** Extra output room for the model's reasoning at each effort (reasoning counts as output). */
|
|
const REASONING_ROOM: Partial<Record<ReasoningEffort, number>> = { low: 8_000, medium: 16_000, high: 32_000, xhigh: 48_000, max: 64_000 };
|
|
|
|
/**
|
|
* The output limit for a call that must fit `answerTokens` of answer: the user's own limit when
|
|
* they set one, otherwise the answer plus room for reasoning at the configured effort, within
|
|
* the model's maximum. A reasoning model at a high effort can otherwise spend the whole limit
|
|
* thinking and be cut off before it answers.
|
|
*/
|
|
export function outputBudget(resolved: ResolvedModel, answerTokens: number): number {
|
|
if (resolved.params.maxOutputTokens) return resolved.params.maxOutputTokens;
|
|
const effort = resolved.params.reasoningEffort;
|
|
const thinks = resolved.model.capabilities.reasoning && effort !== 'none' && effort !== 'minimal';
|
|
const room = !thinks ? 0 : effort ? (REASONING_ROOM[effort] ?? 16_000) : 16_000;
|
|
return Math.min(answerTokens + room, resolved.model.maxOutput ?? answerTokens + room);
|
|
}
|
|
|
|
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
|
|
/** Name + description of the "function" whose arguments are the result. */
|
|
toolName: string;
|
|
toolDescription: string;
|
|
schema: Record<string, unknown>;
|
|
/** Offer the MCP tools allowed in this area before the structured answer. */
|
|
mcpArea?: McpArea;
|
|
}
|
|
|
|
/**
|
|
* Structured output that works on every provider: the schema is offered as a single tool the
|
|
* model must call. Models without tool support (many local ones) get the schema in the prompt
|
|
* and their text is parsed leniently instead.
|
|
*/
|
|
export async function generateStructured(app: AppContext, opts: StructuredOptions): Promise<{ data: unknown; usage: Usage; model: ModelDto; truncated: boolean }> {
|
|
const resolved = resolveModel(app, opts.task, opts.modelId);
|
|
const tool: ToolDef = { name: opts.toolName, description: opts.toolDescription, parameters: opts.schema };
|
|
const promptJson = async () => {
|
|
const system = `${opts.system ?? ''}\n\nRespond with ONLY a JSON object matching this JSON Schema. No prose, no Markdown fences.\n${JSON.stringify(opts.schema)}`.trim();
|
|
const res = await runChat(app, { ...opts, system, jsonMode: true });
|
|
return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
};
|
|
if (!resolved.model.capabilities.tools) return promptJson();
|
|
const chatOpts: ChatOptions = { ...opts, tools: [tool], toolChoice: { name: opts.toolName } };
|
|
let cutOff = false;
|
|
const once = async (maxOutputTokens = chatOpts.maxOutputTokens) => {
|
|
const callOpts = { ...chatOpts, maxOutputTokens };
|
|
const res = opts.mcpArea ? await runChatWithMcp(app, callOpts, opts.mcpArea) : await runChat(app, callOpts);
|
|
const call = res.message.content.find((p) => p.type === 'tool_call');
|
|
if (call && call.type === 'tool_call') return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
|
|
if (res.stopReason === 'max_tokens') {
|
|
cutOff = true;
|
|
throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
|
|
}
|
|
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
|
|
};
|
|
try {
|
|
return await once();
|
|
} catch (err) {
|
|
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
|
|
if (!(err instanceof AIError) || err.code !== 'invalid_response' || opts.signal?.aborted) throw err;
|
|
// Cut off before answering: asking again with the same limit fails the same way, so double it.
|
|
if (cutOff) {
|
|
const doubled = Math.min((chatOpts.maxOutputTokens ?? 16_000) * 2, resolved.model.maxOutput ?? Infinity);
|
|
app.log('warn', `${opts.toolName} ran out of output space; retrying with ${doubled} tokens`);
|
|
return await once(doubled);
|
|
}
|
|
// Malformed output (jsonrepair could not fix it): the model repairs its own JSON when we
|
|
// have it, otherwise it gets one more try at the same request.
|
|
if (err.rawOutput) {
|
|
try {
|
|
return await repairStructured(app, opts, tool, err.rawOutput);
|
|
} catch (repairErr) {
|
|
app.log('warn', `Repairing malformed ${opts.toolName} output failed; asking again`, repairErr instanceof Error ? repairErr.message : repairErr);
|
|
}
|
|
}
|
|
return await once();
|
|
}
|
|
}
|
|
|
|
/** Sends malformed tool-call arguments back to the model to be rewritten as valid JSON. */
|
|
async function repairStructured(app: AppContext, opts: StructuredOptions, tool: ToolDef, raw: string) {
|
|
const res = await runChat(app, {
|
|
task: opts.task,
|
|
purpose: `${opts.purpose ?? 'structured output'} (JSON repair)`,
|
|
modelId: opts.modelId,
|
|
system: 'You repair malformed JSON. Call the tool once with the same content, corrected so it is valid JSON that matches the tool schema. Keep every item and all of its wording; do not add, drop or rewrite content.',
|
|
messages: [{ role: 'user', content: [{ type: 'text', text: `These arguments for "${tool.name}" are not valid JSON:
|
|
|
|
${raw}` }] }],
|
|
tools: [tool],
|
|
toolChoice: { name: tool.name },
|
|
maxOutputTokens: opts.maxOutputTokens,
|
|
signal: opts.signal,
|
|
});
|
|
const call = res.message.content.find((p) => p.type === 'tool_call');
|
|
if (!call || call.type !== 'tool_call') throw new AIError('invalid_response', 'The repair returned no tool call.');
|
|
return { data: call.input as unknown, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
|
}
|
|
|
|
/* ------------------------------ reporting ------------------------------ */
|
|
|
|
export function usageSummary(db: DB, days: number) {
|
|
const since = now() - days * 86_400_000;
|
|
const byModel = db.all<Record<string, any>>(
|
|
`SELECT provider_name AS provider, model, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(cache_read_tokens) AS cached, SUM(output_tokens) AS output, SUM(cost) AS cost,
|
|
SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN cost IS NULL AND ok = 1 THEN 1 ELSE 0 END) AS unpriced
|
|
FROM ai_usage WHERE at >= ? GROUP BY provider_name, model ORDER BY cost DESC, calls DESC`,
|
|
since,
|
|
);
|
|
const byTask = db.all<Record<string, any>>(
|
|
`SELECT task, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost FROM ai_usage WHERE at >= ? AND ok = 1 GROUP BY task ORDER BY cost DESC`,
|
|
since,
|
|
);
|
|
const total = db.get<Record<string, any>>(
|
|
`SELECT COUNT(*) AS calls, COALESCE(SUM(input_tokens), 0) AS input, COALESCE(SUM(cache_read_tokens), 0) AS cached, COALESCE(SUM(output_tokens), 0) AS output, COALESCE(SUM(cost), 0) AS cost FROM ai_usage WHERE at >= ?`,
|
|
since,
|
|
)!;
|
|
const recentErrors = db.all<Record<string, any>>(`SELECT at, provider_name AS provider, model, task, error FROM ai_usage WHERE ok = 0 AND at >= ? ORDER BY at DESC LIMIT 10`, since);
|
|
return { days, total, byModel, byTask, recentErrors };
|
|
}
|
|
|
|
/** True when at least one usable model exists; lets the UI show setup prompts instead of errors. */
|
|
export function aiStatus(app: AppContext) {
|
|
const models = listModels(app.db).filter((m) => m.available);
|
|
const tasks = getTaskSettings(app.db);
|
|
const resolved: Record<string, { modelId: string; label: string; provider: string } | null> = {};
|
|
for (const t of TASKS) {
|
|
try {
|
|
const r = resolveModel(app, t);
|
|
resolved[t] = { modelId: r.model.id, label: r.model.label, provider: r.provider.name };
|
|
} catch {
|
|
resolved[t] = null;
|
|
}
|
|
}
|
|
return { ready: models.length > 0, modelCount: models.length, tasks, resolved };
|
|
}
|