Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
1000
server/ai/assistant.ts
Normal file
1000
server/ai/assistant.ts
Normal file
File diff suppressed because it is too large
Load diff
875
server/ai/generate.ts
Normal file
875
server/ai/generate.ts
Normal file
|
|
@ -0,0 +1,875 @@
|
|||
/**
|
||||
* Content generation: quizzes, flashcards and study guides from documents, pasted text,
|
||||
* existing decks/quizzes, weak areas, or a plain topic.
|
||||
*
|
||||
* Every job writes into a draft row. Nothing reaches the library until the user has
|
||||
* reviewed the draft and saved it. Long jobs are split into batches, each with its own
|
||||
* slice of source material, so output limits and context windows are never a problem.
|
||||
*/
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
validateDraftCard,
|
||||
validateDraftQuestion,
|
||||
type ContentDraft,
|
||||
type DeckDraft,
|
||||
type DraftCard,
|
||||
type DraftQuestion,
|
||||
type NoteDraft,
|
||||
type QuizDraft,
|
||||
} from '../../shared/content.ts';
|
||||
import { draftsFromValue } from '../../shared/importer.ts';
|
||||
import type { DB } from '../db.ts';
|
||||
import { badRequest, notFound, type AppContext } from '../http.ts';
|
||||
import { missedQuestionIds } from '../services/attempts.ts';
|
||||
import {
|
||||
createDeck,
|
||||
createNote,
|
||||
createQuiz,
|
||||
draftCardToInput,
|
||||
draftQuestionToInput,
|
||||
getClass,
|
||||
getDeck,
|
||||
getQuiz,
|
||||
insertCards,
|
||||
insertQuestions,
|
||||
listCards,
|
||||
listQuestions,
|
||||
updateQuestion,
|
||||
} from '../services/content.ts';
|
||||
import { chunkById, formatExcerpts, searchChunks, selectContext, sourceLabel, type ChunkHit } from '../services/documents.ts';
|
||||
import { weakTopics } from '../services/stats.ts';
|
||||
import { newId, now, parseJson } from '../util.ts';
|
||||
import { generateStructured, resolveModel, runChat } from './service.ts';
|
||||
import { AIError } from './types.ts';
|
||||
|
||||
/* ------------------------------ drafts ------------------------------ */
|
||||
|
||||
interface DraftRow {
|
||||
id: string;
|
||||
kind: 'quiz' | 'deck' | 'note';
|
||||
class_id: string | null;
|
||||
folder_id: string | null;
|
||||
title: string;
|
||||
payload: string;
|
||||
meta: string;
|
||||
status: 'generating' | 'ready' | 'failed' | 'saved' | 'discarded';
|
||||
progress: number;
|
||||
progress_label: string;
|
||||
error: string;
|
||||
saved_kind: string | null;
|
||||
saved_id: string | null;
|
||||
conversation_id: string | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
export interface DraftMeta {
|
||||
origin?: 'generate' | 'assistant' | 'convert' | 'improve' | 'import';
|
||||
model?: string;
|
||||
request?: Record<string, unknown>;
|
||||
sources?: { documentId: string; title: string }[];
|
||||
/** For "improve" drafts: the stored question each draft question replaces (same order). */
|
||||
replaceQuestionIds?: string[];
|
||||
/** Suggested target when the draft is meant to extend an existing item. */
|
||||
targetKind?: 'quiz' | 'deck';
|
||||
targetId?: string;
|
||||
notes?: string[];
|
||||
}
|
||||
|
||||
function emptyPayload(kind: DraftRow['kind'], title = ''): ContentDraft {
|
||||
if (kind === 'quiz') return { kind: 'quiz', title, description: '', questions: [] };
|
||||
if (kind === 'deck') return { kind: 'deck', title, description: '', cards: [] };
|
||||
return { kind: 'note', title, content: '' };
|
||||
}
|
||||
|
||||
export function mapDraft(row: DraftRow) {
|
||||
const payload = parseJson<ContentDraft>(row.payload, emptyPayload(row.kind, row.title));
|
||||
return {
|
||||
id: row.id,
|
||||
kind: row.kind,
|
||||
classId: row.class_id,
|
||||
folderId: row.folder_id,
|
||||
title: row.title,
|
||||
payload,
|
||||
meta: parseJson<DraftMeta>(row.meta, {}),
|
||||
status: row.status,
|
||||
progress: row.progress,
|
||||
progressLabel: row.progress_label,
|
||||
error: row.error,
|
||||
savedKind: row.saved_kind,
|
||||
savedId: row.saved_id,
|
||||
conversationId: row.conversation_id,
|
||||
itemCount: payload.kind === 'quiz' ? payload.questions.length : payload.kind === 'deck' ? payload.cards.length : payload.content.length,
|
||||
createdAt: row.created_at,
|
||||
updatedAt: row.updated_at,
|
||||
};
|
||||
}
|
||||
export type DraftDto = ReturnType<typeof mapDraft>;
|
||||
|
||||
export function getDraft(db: DB, id: string): DraftDto {
|
||||
const row = db.get<DraftRow>('SELECT * FROM drafts WHERE id = ?', id);
|
||||
if (!row) throw notFound('Draft');
|
||||
return mapDraft(row);
|
||||
}
|
||||
|
||||
export function createDraft(
|
||||
db: DB,
|
||||
input: { kind: DraftRow['kind']; classId?: string | null; folderId?: string | null; title: string; payload?: ContentDraft; meta?: DraftMeta; status?: DraftRow['status']; conversationId?: string | null },
|
||||
): string {
|
||||
const id = newId();
|
||||
const at = now();
|
||||
db.run(
|
||||
`INSERT INTO drafts (id, kind, class_id, folder_id, title, payload, meta, status, conversation_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
id,
|
||||
input.kind,
|
||||
input.classId ?? null,
|
||||
input.folderId ?? null,
|
||||
input.title.slice(0, 200),
|
||||
JSON.stringify(input.payload ?? emptyPayload(input.kind, input.title)),
|
||||
JSON.stringify(input.meta ?? {}),
|
||||
input.status ?? 'ready',
|
||||
input.conversationId ?? null,
|
||||
at,
|
||||
at,
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
export function listDrafts(db: DB) {
|
||||
return db
|
||||
.all<DraftRow>(`SELECT * FROM drafts WHERE status IN ('generating', 'ready', 'failed') ORDER BY updated_at DESC LIMIT 50`)
|
||||
.map(mapDraft)
|
||||
.map(({ payload: _payload, ...rest }) => rest);
|
||||
}
|
||||
|
||||
export function updateDraftPayload(db: DB, id: string, payload: ContentDraft, extra: { classId?: string | null; folderId?: string | null } = {}): DraftDto {
|
||||
const draft = getDraft(db, id);
|
||||
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
|
||||
if (payload.kind !== draft.kind) throw badRequest('Draft type cannot change');
|
||||
db.run(
|
||||
'UPDATE drafts SET payload = ?, title = ?, class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?',
|
||||
JSON.stringify(payload),
|
||||
(payload.title ?? '').slice(0, 200),
|
||||
extra.classId === undefined ? draft.classId : extra.classId,
|
||||
extra.folderId === undefined ? draft.folderId : extra.folderId,
|
||||
now(),
|
||||
id,
|
||||
);
|
||||
return getDraft(db, id);
|
||||
}
|
||||
|
||||
export const draftSaveSchema = z.object({
|
||||
classId: z.string().optional(),
|
||||
folderId: z.string().nullish(),
|
||||
/** new: create an item. append: add to an existing quiz/deck. replace: apply improved questions in place. */
|
||||
mode: z.enum(['new', 'append', 'replace']).default('new'),
|
||||
targetId: z.string().optional(),
|
||||
});
|
||||
|
||||
/** Turns a reviewed draft into real content. Invalid items are rejected, never silently dropped. */
|
||||
export function saveDraft(app: AppContext, id: string, input: z.input<typeof draftSaveSchema>): { kind: string; id: string; count: number } {
|
||||
const { db } = app;
|
||||
const opts = draftSaveSchema.parse(input);
|
||||
const draft = getDraft(db, id);
|
||||
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
|
||||
if (draft.status === 'saved' && draft.savedId) return { kind: draft.savedKind ?? draft.kind, id: draft.savedId, count: 0 };
|
||||
const payload = draft.payload;
|
||||
const classId = opts.classId ?? draft.classId;
|
||||
let result: { kind: string; id: string; count: number };
|
||||
|
||||
if (payload.kind === 'quiz') {
|
||||
const bad = payload.questions.flatMap((q, i) => validateDraftQuestion(q, i)).filter((x) => x.level === 'error');
|
||||
if (bad.length > 0) throw badRequest(`Question ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
|
||||
if (payload.questions.length === 0) throw badRequest('There are no questions to save');
|
||||
const inputs = payload.questions.map(draftQuestionToInput);
|
||||
if (opts.mode === 'replace') {
|
||||
const ids = draft.meta.replaceQuestionIds ?? [];
|
||||
const quizId = opts.targetId ?? draft.meta.targetId;
|
||||
if (!quizId) throw badRequest('No quiz to update');
|
||||
getQuiz(db, quizId);
|
||||
db.tx(() => {
|
||||
inputs.forEach((q, i) => {
|
||||
const target = ids[i];
|
||||
if (target && db.get('SELECT id FROM questions WHERE id = ? AND quiz_id = ?', target, quizId)) updateQuestion(db, target, q);
|
||||
else insertQuestions(db, quizId, [q]);
|
||||
});
|
||||
});
|
||||
result = { kind: 'quiz', id: quizId, count: inputs.length };
|
||||
} else if (opts.mode === 'append') {
|
||||
const quizId = opts.targetId ?? draft.meta.targetId;
|
||||
if (!quizId) throw badRequest('Choose a quiz to add these questions to');
|
||||
getQuiz(db, quizId);
|
||||
insertQuestions(db, quizId, inputs);
|
||||
result = { kind: 'quiz', id: quizId, count: inputs.length };
|
||||
} else {
|
||||
if (!classId) throw badRequest('Choose a class to save this quiz in');
|
||||
const quiz = createQuiz(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled quiz', description: payload.description, questions: inputs });
|
||||
result = { kind: 'quiz', id: quiz.id, count: inputs.length };
|
||||
}
|
||||
} else if (payload.kind === 'deck') {
|
||||
const bad = payload.cards.flatMap((c, i) => validateDraftCard(c, i)).filter((x) => x.level === 'error');
|
||||
if (bad.length > 0) throw badRequest(`Card ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
|
||||
if (payload.cards.length === 0) throw badRequest('There are no cards to save');
|
||||
const inputs = payload.cards.map(draftCardToInput);
|
||||
if (opts.mode === 'append') {
|
||||
const deckId = opts.targetId ?? draft.meta.targetId;
|
||||
if (!deckId) throw badRequest('Choose a deck to add these cards to');
|
||||
getDeck(db, deckId);
|
||||
insertCards(db, deckId, inputs);
|
||||
result = { kind: 'deck', id: deckId, count: inputs.length };
|
||||
} else {
|
||||
if (!classId) throw badRequest('Choose a class to save this deck in');
|
||||
const deck = createDeck(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled deck', description: payload.description, cards: inputs });
|
||||
result = { kind: 'deck', id: deck.id, count: inputs.length };
|
||||
}
|
||||
} else {
|
||||
if (!payload.content.trim()) throw badRequest('The note is empty');
|
||||
if (!classId) throw badRequest('Choose a class to save this note in');
|
||||
const note = createNote(db, {
|
||||
classId,
|
||||
folderId: opts.folderId ?? draft.folderId,
|
||||
title: payload.title.trim() || 'Untitled note',
|
||||
content: payload.content,
|
||||
kind: 'guide',
|
||||
source: { model: draft.meta.model, sources: draft.meta.sources },
|
||||
});
|
||||
result = { kind: 'note', id: note.id, count: 1 };
|
||||
}
|
||||
db.run(`UPDATE drafts SET status = 'saved', saved_kind = ?, saved_id = ?, updated_at = ? WHERE id = ?`, result.kind, result.id, now(), id);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function discardDraft(db: DB, id: string): void {
|
||||
cancelGeneration(id);
|
||||
const { changes } = db.run(`UPDATE drafts SET status = 'discarded', updated_at = ? WHERE id = ?`, now(), id);
|
||||
if (!changes) throw notFound('Draft');
|
||||
}
|
||||
|
||||
/** Removes finished drafts after a while and marks jobs interrupted by a restart as failed. */
|
||||
export function tidyDrafts(db: DB): void {
|
||||
db.run(`UPDATE drafts SET status = 'failed', error = 'Generation was interrupted by a restart. Start it again.' WHERE status = 'generating'`);
|
||||
db.run(`DELETE FROM drafts WHERE status IN ('saved', 'discarded') AND updated_at < ?`, now() - 14 * 86_400_000);
|
||||
db.run(`DELETE FROM drafts WHERE status IN ('ready', 'failed') AND updated_at < ?`, now() - 60 * 86_400_000);
|
||||
}
|
||||
|
||||
/* ------------------------------ request ------------------------------ */
|
||||
|
||||
export const generateRequestSchema = z.object({
|
||||
kind: z.enum(['quiz', 'deck', 'note']),
|
||||
classId: z.string().nullish(),
|
||||
folderId: z.string().nullish(),
|
||||
title: z.string().trim().max(200).optional(),
|
||||
source: z
|
||||
.object({
|
||||
documentIds: z.array(z.string()).max(50).optional(),
|
||||
/** Use every processed document in the class. */
|
||||
allClassDocuments: z.boolean().optional(),
|
||||
text: z.string().max(400_000).optional(),
|
||||
quizId: z.string().optional(),
|
||||
deckId: z.string().optional(),
|
||||
noteId: z.string().optional(),
|
||||
/** Build from the questions the user keeps missing. */
|
||||
weakAreas: z.boolean().optional(),
|
||||
})
|
||||
.default({}),
|
||||
/** Topic or focus, e.g. "pediatric respiratory disorders". */
|
||||
focus: z.string().trim().max(600).optional(),
|
||||
instructions: z.string().trim().max(4000).optional(),
|
||||
count: z.number().int().min(1).max(200).optional(),
|
||||
style: z.enum(['nclex', 'standard', 'recall']).optional(),
|
||||
difficulty: z.enum(['easy', 'medium', 'hard', 'mixed']).optional(),
|
||||
/** Share of select-all-that-apply questions, 0..1. */
|
||||
sataShare: z.number().min(0).max(1).optional(),
|
||||
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
|
||||
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
|
||||
modelId: z.string().nullish(),
|
||||
});
|
||||
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
|
||||
|
||||
/* ------------------------------ prompts ------------------------------ */
|
||||
|
||||
const QUESTION_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
questions: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
type: { type: 'string', enum: ['single', 'multi'], description: 'single = one correct answer; multi = select all that apply' },
|
||||
stem: { type: 'string', description: 'The question. Markdown allowed.' },
|
||||
options: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
text: { type: 'string' },
|
||||
correct: { type: 'boolean' },
|
||||
why: { type: 'string', description: 'One sentence on why this choice is right or wrong.' },
|
||||
},
|
||||
required: ['text', 'correct', 'why'],
|
||||
},
|
||||
},
|
||||
rationale: { type: 'string', description: 'Teaching explanation of the correct answer. Markdown allowed.' },
|
||||
topic: { type: 'string', description: 'Short topic label, reused across related questions (2-4 words).' },
|
||||
difficulty: { type: 'string', enum: ['easy', 'medium', 'hard'] },
|
||||
source: { type: 'string', description: 'Id of the excerpt this question is based on, e.g. "c42". Empty if none.' },
|
||||
},
|
||||
required: ['type', 'stem', 'options', 'rationale', 'topic', 'difficulty'],
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['questions'],
|
||||
};
|
||||
|
||||
const CARD_SCHEMA = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
cards: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
front: { type: 'string', description: 'Prompt side: one specific question or term.' },
|
||||
back: { type: 'string', description: 'Answer side. Markdown allowed.' },
|
||||
hint: { type: 'string', description: 'Optional memory hook or mnemonic. Empty string if none.' },
|
||||
tags: { type: 'array', items: { type: 'string' }, description: '1-2 short topic tags.' },
|
||||
},
|
||||
required: ['front', 'back'],
|
||||
},
|
||||
},
|
||||
},
|
||||
required: ['cards'],
|
||||
};
|
||||
|
||||
const NCLEX_RULES = `Write NCLEX-style (Next Generation NCLEX) items:
|
||||
- Test clinical judgment at the application and analysis level: recognise cues, analyse, prioritise, choose and evaluate actions. Avoid pure recall.
|
||||
- Open with a brief, realistic client scenario containing only relevant data (age, diagnosis, key findings, vital signs or lab values with units).
|
||||
- Ask one clear question. Use priority wording where it fits (first, best, most important, requires immediate follow-up) and build it on a real framework: ABCs, Maslow, the nursing process, acute before chronic, unstable before stable, safety, least invasive first.
|
||||
- Cover the client-needs areas where the material allows: safe and effective care, pharmacology, physiological adaptation, reduction of risk, health promotion, psychosocial integrity, delegation and assignment.
|
||||
- Every distractor must be plausible to a student with partial knowledge and wrong for a specific reason. Keep choices parallel in grammar and similar in length. Never use "all of the above" or "none of the above". Do not give the answer away through wording or length.
|
||||
- Multiple-choice items have exactly 4 choices with exactly one best answer. Select-all-that-apply items have 5 or 6 choices with 2 to 5 correct, and the stem ends with "Select all that apply."
|
||||
- Use generic drug names. Use current, standard practice.
|
||||
- The rationale teaches: state why the correct answer is correct, name the principle behind it, and give each choice its own "why".`;
|
||||
|
||||
const STANDARD_RULES = `Write clear exam-quality questions:
|
||||
- Each question tests one idea and has one defensible answer set.
|
||||
- Mix understanding and application; avoid trivia and trick wording.
|
||||
- Distractors are plausible and parallel. Never use "all of the above" or "none of the above".
|
||||
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
|
||||
- The rationale explains the correct answer, and each choice gets its own "why".`;
|
||||
|
||||
const RECALL_RULES = `Write straightforward recall questions that check key facts, definitions, normal values and classifications.
|
||||
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
|
||||
- Keep stems short and unambiguous. Each choice gets a brief "why".`;
|
||||
|
||||
const CARD_RULES: Record<string, string> = {
|
||||
concise: 'Each card tests exactly one fact. Fronts are specific questions or terms; backs are short (a phrase or one or two lines).',
|
||||
detailed: 'Each card covers one concept. Backs may use a short bullet list (up to 5 bullets) for the key points.',
|
||||
medication:
|
||||
'Make medication cards. Front: the generic drug name (with a common brand name in parentheses when well known). Back: a Markdown list with **Class**, **Action**, **Uses**, **Adverse effects** (the most important ones), **Nursing considerations** and **Client teaching**. Only include what the material supports.',
|
||||
definition: 'Make term and definition cards. Front: the term. Back: a precise definition in one or two sentences, plus an example when useful.',
|
||||
};
|
||||
|
||||
const NOTE_RULES: Record<string, string> = {
|
||||
study_guide:
|
||||
'Write a study guide in Markdown: a one-paragraph overview, then sections with headings. Under each heading give the key concepts as tight bullet points, important values and medications in tables where that helps, common pitfalls, and finish with a "Check yourself" list of 5-10 self-test questions (answers at the very end).',
|
||||
summary: 'Write a faithful summary in Markdown: a short overview paragraph, then the main points grouped under headings. Keep it compact and do not add outside information.',
|
||||
outline: 'Write a hierarchical outline in Markdown using nested bullet points. Capture the structure and every key point of the material without full sentences where a phrase will do.',
|
||||
cheat_sheet: 'Write a one-page cheat sheet in Markdown: dense, scannable, organised under short headings, using tables for values, drug classes and comparisons, and bold for must-know facts.',
|
||||
};
|
||||
|
||||
function baseSystem(req: GenerateRequest, hasSources: boolean): string {
|
||||
const lines = [
|
||||
'You are an expert educator writing study material for a nursing student. Accuracy is critical: this material is used to prepare for licensing exams.',
|
||||
hasSources
|
||||
? 'Base everything on the provided source material. Do not introduce facts that contradict it. You may rely on standard, well-established knowledge only to write plausible distractors and clear explanations.'
|
||||
: 'No source material was provided, so rely on current, well-established knowledge. If something is uncertain or varies by institution, prefer the most widely taught standard.',
|
||||
'Write in clear, natural English. Use Markdown inside text fields where it aids clarity (bold key terms, short lists, tables). Do not wrap output in code fences.',
|
||||
];
|
||||
if (req.instructions) lines.push(`Additional instructions from the student (follow them unless they conflict with accuracy):\n${req.instructions}`);
|
||||
return lines.join('\n\n');
|
||||
}
|
||||
|
||||
/* ------------------------------ source gathering ------------------------------ */
|
||||
|
||||
interface Sources {
|
||||
/** Batches of excerpts (documents). */
|
||||
groups: ChunkHit[][];
|
||||
/** Free text used as-is (pasted notes, an existing deck or quiz, missed questions). */
|
||||
inline: string;
|
||||
docs: { documentId: string; title: string }[];
|
||||
refs: Map<string, string>;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Sources {
|
||||
const { db } = app;
|
||||
const src = req.source;
|
||||
const out: Sources = { groups: [], inline: '', docs: [], refs: new Map(), totalTokens: 0 };
|
||||
const inline: string[] = [];
|
||||
|
||||
let documentIds = src.documentIds ?? [];
|
||||
if (src.allClassDocuments && req.classId) {
|
||||
documentIds = db.all<{ id: string }>(`SELECT id FROM documents WHERE class_id = ? AND status = 'ready' AND chunk_count > 0 ORDER BY sort_order, created_at`, req.classId).map((d) => d.id);
|
||||
}
|
||||
if (documentIds.length > 0) {
|
||||
const rows = db.all<{ id: string; title: string; status: string; chunk_count: number }>(
|
||||
`SELECT id, title, status, chunk_count FROM documents WHERE id IN (${documentIds.map(() => '?').join(',')})`,
|
||||
...documentIds,
|
||||
);
|
||||
const usable = rows.filter((r) => r.status === 'ready' && r.chunk_count > 0);
|
||||
if (usable.length === 0) {
|
||||
throw badRequest(rows.some((r) => r.status !== 'ready') ? 'Those documents are still being processed. Try again in a moment.' : 'Those documents have no readable text yet.');
|
||||
}
|
||||
out.docs = usable.map((r) => ({ documentId: r.id, title: r.title }));
|
||||
out.groups = selectContext(db, { documentIds: usable.map((r) => r.id) }, { focus: req.focus, budgetTokens, groups });
|
||||
}
|
||||
|
||||
if (src.text?.trim()) inline.push(`Notes provided by the student:\n${src.text.trim()}`);
|
||||
if (src.noteId) {
|
||||
const note = db.get<{ title: string; content: string }>('SELECT title, content FROM notes WHERE id = ?', src.noteId);
|
||||
if (note) inline.push(`Note "${note.title}":\n${note.content}`);
|
||||
}
|
||||
if (src.deckId) {
|
||||
const deck = getDeck(db, src.deckId);
|
||||
const cards = listCards(db, deck.id);
|
||||
if (cards.length === 0) throw badRequest('That deck has no cards');
|
||||
inline.push(`Flashcards from the deck "${deck.title}":\n${cards.map((c, i) => `${i + 1}. ${c.front}\n -> ${c.back}`).join('\n')}`);
|
||||
}
|
||||
if (src.quizId) {
|
||||
const quiz = getQuiz(db, src.quizId);
|
||||
const questions = listQuestions(db, quiz.id);
|
||||
if (questions.length === 0) throw badRequest('That quiz has no questions');
|
||||
inline.push(
|
||||
`Questions from the quiz "${quiz.title}":\n${questions
|
||||
.map((q, i) => `${i + 1}. ${q.stem}\n Correct: ${q.options.filter((o) => o.correct).map((o) => o.text).join('; ')}\n Rationale: ${q.rationale}`)
|
||||
.join('\n')}`,
|
||||
);
|
||||
}
|
||||
if (src.weakAreas) {
|
||||
const ids = missedQuestionIds(db, { classId: req.classId ?? undefined }, 40);
|
||||
const topics = weakTopics(db, { classId: req.classId ?? undefined }, 10);
|
||||
if (ids.length === 0 && topics.length === 0) throw badRequest('There is not enough quiz history yet to find weak areas. Take a few quizzes first.');
|
||||
const missed = ids
|
||||
.map((id) => db.get<{ stem: string; topic: string; rationale: string }>('SELECT stem, topic, rationale FROM questions WHERE id = ?', id))
|
||||
.filter((q): q is { stem: string; topic: string; rationale: string } => Boolean(q));
|
||||
inline.push(
|
||||
[
|
||||
'The student keeps missing these concepts. Write NEW items that test the same underlying knowledge from different angles. Do not repeat or lightly reword the original questions.',
|
||||
topics.length ? `Weakest topics: ${topics.map((t) => `${t.topic} (${Math.round(t.accuracy * 100)}% correct)`).join(', ')}` : '',
|
||||
missed.length ? `Questions answered incorrectly:\n${missed.map((q, i) => `${i + 1}. [${q.topic || 'General'}] ${q.stem}\n Key point: ${q.rationale.slice(0, 400)}`).join('\n')}` : '',
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n\n'),
|
||||
);
|
||||
// Pull supporting passages from the class documents for those topics.
|
||||
if (req.classId && out.groups.length === 0) {
|
||||
const query = [...topics.map((t) => t.topic), ...missed.slice(0, 10).map((q) => q.topic)].filter(Boolean).join(' ');
|
||||
const hits = query ? searchChunks(db, query, { classId: req.classId }, 24) : [];
|
||||
if (hits.length > 0) {
|
||||
const perGroup = Math.max(1, Math.ceil(hits.length / groups));
|
||||
for (let i = 0; i < hits.length; i += perGroup) out.groups.push(hits.slice(i, i + perGroup));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out.inline = inline.join('\n\n');
|
||||
for (const g of out.groups) for (const c of g) out.refs.set(`c${c.id}`, sourceLabel(c));
|
||||
out.totalTokens = out.groups.reduce((n, g) => n + g.reduce((m, c) => m + c.tokens, 0), 0) + Math.ceil(out.inline.length / 4);
|
||||
return out;
|
||||
}
|
||||
|
||||
function sourceBlock(sources: Sources, groupIndex: number): string {
|
||||
const parts: string[] = [];
|
||||
const group = sources.groups.length > 0 ? sources.groups[groupIndex % sources.groups.length] : [];
|
||||
if (group.length > 0) parts.push(`SOURCE EXCERPTS (each starts with its id in brackets):\n\n${formatExcerpts(group)}`);
|
||||
if (sources.inline) parts.push(sources.inline);
|
||||
return parts.join('\n\n======\n\n');
|
||||
}
|
||||
|
||||
/* ------------------------------ job control ------------------------------ */
|
||||
|
||||
const running = new Map<string, AbortController>();
|
||||
|
||||
export function cancelGeneration(draftId: string): void {
|
||||
running.get(draftId)?.abort();
|
||||
running.delete(draftId);
|
||||
}
|
||||
|
||||
function setProgress(db: DB, id: string, progress: number, label: string, payload?: ContentDraft): void {
|
||||
if (payload) {
|
||||
db.run('UPDATE drafts SET progress = ?, progress_label = ?, payload = ?, updated_at = ? WHERE id = ?', progress, label, JSON.stringify(payload), now(), id);
|
||||
} else {
|
||||
db.run('UPDATE drafts SET progress = ?, progress_label = ?, updated_at = ? WHERE id = ?', progress, label, now(), id);
|
||||
}
|
||||
}
|
||||
|
||||
function defaultTitle(app: AppContext, req: GenerateRequest, sources: Sources): string {
|
||||
if (req.title) return req.title;
|
||||
const subject =
|
||||
req.focus?.replace(/\s+/g, ' ').slice(0, 80) ||
|
||||
(sources.docs.length === 1 ? sources.docs[0].title : '') ||
|
||||
(req.source.deckId ? getDeck(app.db, req.source.deckId).title : '') ||
|
||||
(req.source.quizId ? getQuiz(app.db, req.source.quizId).title : '') ||
|
||||
(req.source.weakAreas ? 'Weak areas' : '') ||
|
||||
(req.classId ? getClass(app.db, req.classId).name : 'Untitled');
|
||||
const suffix = req.kind === 'quiz' ? (req.source.weakAreas ? 'practice' : req.style === 'nclex' ? 'NCLEX practice' : 'quiz') : req.kind === 'deck' ? 'flashcards' : NOTE_TITLES[req.noteStyle ?? 'study_guide'];
|
||||
const cap = subject.charAt(0).toUpperCase() + subject.slice(1);
|
||||
return `${cap} · ${suffix}`.slice(0, 200);
|
||||
}
|
||||
|
||||
const NOTE_TITLES: Record<string, string> = { study_guide: 'study guide', summary: 'summary', outline: 'outline', cheat_sheet: 'cheat sheet' };
|
||||
|
||||
/**
|
||||
* Starts a generation job in the background and returns the draft id immediately.
|
||||
* The caller polls the draft (or awaits `done`) for progress.
|
||||
*/
|
||||
export function startGeneration(app: AppContext, input: z.input<typeof generateRequestSchema>, opts: { conversationId?: string | null; origin?: DraftMeta['origin'] } = {}): { draftId: string; done: Promise<DraftDto> } {
|
||||
const req = generateRequestSchema.parse(input);
|
||||
if (req.classId) getClass(app.db, req.classId);
|
||||
// Fail fast (before creating a draft) when no model is configured.
|
||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
||||
const draftId = createDraft(app.db, {
|
||||
kind: req.kind,
|
||||
classId: req.classId ?? null,
|
||||
folderId: req.folderId ?? null,
|
||||
title: req.title ?? '',
|
||||
status: 'generating',
|
||||
conversationId: opts.conversationId ?? null,
|
||||
meta: { origin: opts.origin ?? 'generate', model: resolved.model.label, request: { ...req, source: { ...req.source, text: req.source.text ? `${req.source.text.slice(0, 200)}…` : undefined } } },
|
||||
});
|
||||
const controller = new AbortController();
|
||||
running.set(draftId, controller);
|
||||
const done = runGeneration(app, draftId, req, controller.signal)
|
||||
.catch((err) => {
|
||||
const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.';
|
||||
if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err);
|
||||
const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId);
|
||||
if (current?.status === 'generating') {
|
||||
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId);
|
||||
}
|
||||
})
|
||||
.then(() => {
|
||||
running.delete(draftId);
|
||||
return getDraft(app.db, draftId);
|
||||
});
|
||||
return { draftId, done };
|
||||
}
|
||||
|
||||
async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise<void> {
|
||||
const { db } = app;
|
||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
||||
const window = resolved.model.contextWindow ?? 32_000;
|
||||
const budgetTokens = Math.max(2_500, Math.min(Math.floor(window * 0.4), 28_000));
|
||||
setProgress(db, draftId, 0.03, 'Reading your material');
|
||||
|
||||
if (req.kind === 'note') {
|
||||
await generateNote(app, draftId, req, signal, budgetTokens, window);
|
||||
return;
|
||||
}
|
||||
|
||||
const isQuiz = req.kind === 'quiz';
|
||||
const perBatch = isQuiz ? 8 : 25;
|
||||
const defaultCount = isQuiz ? 15 : 30;
|
||||
let count = req.count ?? defaultCount;
|
||||
if ((req.source.deckId || req.source.quizId) && !req.count) {
|
||||
// Conversions default to one item per source item.
|
||||
count = req.source.deckId ? listCards(db, req.source.deckId).length : listQuestions(db, req.source.quizId!).length;
|
||||
count = Math.min(Math.max(count, 1), 100);
|
||||
}
|
||||
const batches = Math.ceil(count / perBatch);
|
||||
const sources = gatherSources(app, req, budgetTokens, batches);
|
||||
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
|
||||
if (!hasSources && !req.focus && !req.instructions) throw badRequest('Choose source material or describe a topic to generate from.');
|
||||
|
||||
const title = defaultTitle(app, req, sources);
|
||||
const draft: QuizDraft | DeckDraft = isQuiz ? { kind: 'quiz', title, description: '', questions: [] } : { kind: 'deck', title, description: '', cards: [] };
|
||||
const items = (): (DraftQuestion | DraftCard)[] => (draft.kind === 'quiz' ? draft.questions : draft.cards);
|
||||
const key = (x: DraftQuestion | DraftCard) => ('stem' in x ? x.stem : x.front).toLowerCase().replace(/\W+/g, ' ').trim().slice(0, 120);
|
||||
const seen = new Set<string>();
|
||||
const notes: string[] = [];
|
||||
|
||||
const style = req.style ?? 'nclex';
|
||||
const rules = isQuiz ? (style === 'nclex' ? NCLEX_RULES : style === 'recall' ? RECALL_RULES : STANDARD_RULES) : CARD_RULES[req.cardStyle ?? 'concise'];
|
||||
const system = `${baseSystem(req, hasSources)}\n\n${rules}`;
|
||||
const sata = req.sataShare ?? (style === 'nclex' ? 0.3 : 0.2);
|
||||
const difficulty = req.difficulty ?? 'mixed';
|
||||
|
||||
const runBatch = async (index: number, want: number): Promise<number> => {
|
||||
const written = items().map((x) => `- ${('stem' in x ? x.stem : x.front).replace(/\s+/g, ' ').slice(0, 140)}`);
|
||||
const sataCount = Math.round(want * sata);
|
||||
const ask = isQuiz
|
||||
? [
|
||||
`Write ${want} ${style === 'nclex' ? 'NCLEX-style ' : ''}questions${req.focus ? ` about: ${req.focus}` : ''}.`,
|
||||
sata <= 0 ? 'Use only multiple-choice (single) questions.' : sata >= 1 ? 'Use only select-all-that-apply (multi) questions.' : `About ${sataCount} of them should be select-all-that-apply (multi); the rest multiple choice (single).`,
|
||||
difficulty === 'mixed' ? 'Mix difficulty: mostly medium, with a few easy and a few hard.' : `Difficulty: ${difficulty}.`,
|
||||
sources.groups.length > 0 ? 'Spread the questions across the excerpts instead of clustering on one. Set "source" to the id of the excerpt each question is based on.' : '',
|
||||
req.source.deckId ? 'Turn the flashcards into questions: use each card\'s answer as the correct choice and write distractors that a student could confuse with it.' : '',
|
||||
]
|
||||
: [
|
||||
`Write ${want} flashcards${req.focus ? ` about: ${req.focus}` : ''}.`,
|
||||
sources.groups.length > 0 ? 'Cover the most testable facts across all of the excerpts.' : '',
|
||||
req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states it with the essential reasoning.' : '',
|
||||
];
|
||||
if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`);
|
||||
const user = [sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n');
|
||||
|
||||
const result = await generateStructured(app, {
|
||||
task: 'generate',
|
||||
purpose: isQuiz ? 'quiz generation' : 'flashcard generation',
|
||||
modelId: req.modelId,
|
||||
system,
|
||||
messages: [{ role: 'user', content: [{ type: 'text', text: user }] }],
|
||||
toolName: isQuiz ? 'submit_questions' : 'submit_flashcards',
|
||||
toolDescription: isQuiz ? 'Submit the finished questions.' : 'Submit the finished flashcards.',
|
||||
schema: isQuiz ? QUESTION_SCHEMA : CARD_SCHEMA,
|
||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
||||
signal,
|
||||
});
|
||||
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === req.kind) as QuizDraft | DeckDraft | undefined;
|
||||
const incoming: (DraftQuestion | DraftCard)[] = parsed ? (parsed.kind === 'quiz' ? parsed.questions : parsed.cards) : [];
|
||||
let added = 0;
|
||||
for (const item of incoming) {
|
||||
const k = key(item);
|
||||
if (!k || seen.has(k)) continue;
|
||||
const problems = ('stem' in item ? validateDraftQuestion(item, 0) : validateDraftCard(item, 0)).filter((p) => p.level === 'error');
|
||||
if (problems.length > 0) continue; // malformed items are dropped here and replaced by a top-up batch
|
||||
if ('stem' in item) {
|
||||
const ref = (item.source ?? '').replace(/[[\]\s]/g, '');
|
||||
item.source = sources.refs.get(ref) ?? undefined;
|
||||
}
|
||||
seen.add(k);
|
||||
(items() as (DraftQuestion | DraftCard)[]).push(item);
|
||||
added += 1;
|
||||
if (items().length >= count) break;
|
||||
}
|
||||
if (result.truncated) notes.push('One batch hit the model output limit, so it produced fewer items than requested.');
|
||||
return added;
|
||||
};
|
||||
|
||||
let failures = 0;
|
||||
for (let i = 0; i < batches && items().length < count; i += 1) {
|
||||
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
|
||||
const want = Math.min(perBatch, count - items().length);
|
||||
setProgress(db, draftId, 0.05 + (0.9 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${items().length + 1}-${Math.min(count, items().length + want)} of ${count}`, draft);
|
||||
try {
|
||||
await runBatch(i, want);
|
||||
} catch (err) {
|
||||
// One bad batch should not sink a long job that already has good content.
|
||||
failures += 1;
|
||||
if (items().length === 0 || failures > 2 || (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code))) throw err;
|
||||
notes.push(`A batch failed and was skipped: ${err instanceof Error ? err.message : 'unknown error'}`);
|
||||
}
|
||||
}
|
||||
// Top up when validation or de-duplication left the draft short.
|
||||
for (let extra = 0; extra < 2 && items().length < count && items().length > 0; extra += 1) {
|
||||
if (signal.aborted) break;
|
||||
const want = Math.min(perBatch, count - items().length);
|
||||
setProgress(db, draftId, 0.92, `Writing the last ${want} ${isQuiz ? 'questions' : 'cards'}`, draft);
|
||||
try {
|
||||
if ((await runBatch(batches + extra, want)) === 0) break;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
|
||||
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
|
||||
|
||||
db.run(
|
||||
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
||||
draft.title,
|
||||
JSON.stringify(draft),
|
||||
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs, notes: [...new Set(notes)] } satisfies DraftMeta),
|
||||
now(),
|
||||
draftId,
|
||||
);
|
||||
}
|
||||
|
||||
async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise<void> {
|
||||
const { db } = app;
|
||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
||||
const style = req.noteStyle ?? 'study_guide';
|
||||
// A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation.
|
||||
const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000));
|
||||
const sources = gatherSources(app, req, perPass, 12);
|
||||
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
|
||||
if (!hasSources && !req.focus) throw badRequest('Choose source material or describe a topic to write about.');
|
||||
const title = defaultTitle(app, req, sources);
|
||||
const system = `${baseSystem(req, hasSources)}\n\n${NOTE_RULES[style]}\n\n${
|
||||
sources.groups.length > 0 ? 'Cite the excerpts you draw on by putting their id in brackets at the end of the relevant line or paragraph, e.g. [c42]. Use only ids that appear in the source.' : ''
|
||||
}\nStart directly with the content (no preamble, no title line).`;
|
||||
|
||||
// Merge excerpt groups into as few passes as fit.
|
||||
const passes: ChunkHit[][] = [];
|
||||
let current: ChunkHit[] = [];
|
||||
let size = 0;
|
||||
for (const g of sources.groups) {
|
||||
const t = g.reduce((n, c) => n + c.tokens, 0);
|
||||
if (current.length > 0 && size + t > perPass) {
|
||||
passes.push(current);
|
||||
current = [];
|
||||
size = 0;
|
||||
}
|
||||
current.push(...g);
|
||||
size += t;
|
||||
}
|
||||
if (current.length > 0) passes.push(current);
|
||||
|
||||
const ask = `${req.focus ? `Focus on: ${req.focus}\n` : ''}Write the ${NOTE_TITLES[style]} now.`;
|
||||
let content = '';
|
||||
if (passes.length <= 1) {
|
||||
setProgress(db, draftId, 0.2, 'Writing');
|
||||
const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n');
|
||||
const res = await runChat(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000, signal });
|
||||
content = res.text;
|
||||
} else {
|
||||
// Long material: condense each part, then write the final piece from the condensed notes.
|
||||
const partials: string[] = [];
|
||||
for (let i = 0; i < passes.length; i += 1) {
|
||||
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
|
||||
setProgress(db, draftId, 0.05 + (0.7 * i) / passes.length, `Reading part ${i + 1} of ${passes.length}`);
|
||||
const res = await runChat(app, {
|
||||
task: 'generate',
|
||||
purpose: 'study guide',
|
||||
modelId: req.modelId,
|
||||
system: `${baseSystem(req, true)}\n\nCondense this part of the material into detailed notes: every key concept, value, medication, cause, sign, intervention and warning, as Markdown bullet points under short headings. Keep the bracketed excerpt ids (e.g. [c42]) next to the points they support. Do not add outside information.`,
|
||||
messages: [{ role: 'user', content: [{ type: 'text', text: `${req.focus ? `Focus on: ${req.focus}\n\n` : ''}SOURCE EXCERPTS (part ${i + 1} of ${passes.length}):\n\n${formatExcerpts(passes[i])}` }] }],
|
||||
maxOutputTokens: 6_000,
|
||||
signal,
|
||||
});
|
||||
partials.push(res.text.trim());
|
||||
}
|
||||
setProgress(db, draftId, 0.8, 'Writing');
|
||||
const res = await runChat(app, {
|
||||
task: 'generate',
|
||||
purpose: 'study guide',
|
||||
modelId: req.modelId,
|
||||
system,
|
||||
messages: [{ role: 'user', content: [{ type: 'text', text: `CONDENSED NOTES FROM THE FULL MATERIAL:\n\n${partials.join('\n\n---\n\n')}\n\n======\n\n${sources.inline ? `${sources.inline}\n\n======\n\n` : ''}${ask}` }] }],
|
||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
||||
signal,
|
||||
});
|
||||
content = res.text;
|
||||
}
|
||||
content = content.trim().replace(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i, '$1');
|
||||
if (!content) throw new AIError('invalid_response', 'The model returned an empty document. Try again.');
|
||||
|
||||
// Turn [c42] markers into readable citations and append a source list.
|
||||
const used = new Map<string, string>();
|
||||
content = content.replace(/\[(c\d+(?:\s*,\s*c\d+)*)\]/g, (_m, list: string) => {
|
||||
const labels = list
|
||||
.split(',')
|
||||
.map((r) => r.trim())
|
||||
.map((r) => {
|
||||
const label = sources.refs.get(r) ?? (() => {
|
||||
const hit = chunkById(db, Number(r.slice(1)));
|
||||
return hit ? sourceLabel(hit) : null;
|
||||
})();
|
||||
if (label) used.set(label, label);
|
||||
return label;
|
||||
})
|
||||
.filter(Boolean);
|
||||
return labels.length ? ` *(${[...new Set(labels)].join('; ')})*` : '';
|
||||
});
|
||||
const payload: NoteDraft = { kind: 'note', title, content };
|
||||
db.run(
|
||||
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
|
||||
title,
|
||||
JSON.stringify(payload),
|
||||
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs } satisfies DraftMeta),
|
||||
now(),
|
||||
draftId,
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------ improve existing questions ------------------------------ */
|
||||
|
||||
export const improveRequestSchema = z.object({
|
||||
quizId: z.string(),
|
||||
questionIds: z.array(z.string()).max(60).optional(),
|
||||
instructions: z.string().trim().max(2000).optional(),
|
||||
modelId: z.string().nullish(),
|
||||
});
|
||||
|
||||
/** Rewrites existing questions (clearer stems, better distractors, fuller rationales) into a draft the user can apply. */
|
||||
export function startImprove(app: AppContext, input: z.input<typeof improveRequestSchema>, opts: { conversationId?: string | null } = {}): { draftId: string; done: Promise<DraftDto> } {
|
||||
const req = improveRequestSchema.parse(input);
|
||||
const quiz = getQuiz(app.db, req.quizId);
|
||||
let questions = listQuestions(app.db, quiz.id);
|
||||
if (req.questionIds?.length) questions = questions.filter((q) => req.questionIds!.includes(q.id));
|
||||
if (questions.length === 0) throw badRequest('No questions to improve');
|
||||
questions = questions.slice(0, 60);
|
||||
const resolved = resolveModel(app, 'generate', req.modelId);
|
||||
const draftId = createDraft(app.db, {
|
||||
kind: 'quiz',
|
||||
classId: quiz.class_id,
|
||||
title: quiz.title,
|
||||
status: 'generating',
|
||||
conversationId: opts.conversationId ?? null,
|
||||
meta: { origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: [] },
|
||||
});
|
||||
const controller = new AbortController();
|
||||
running.set(draftId, controller);
|
||||
const done = (async () => {
|
||||
const out: DraftQuestion[] = [];
|
||||
const ids: string[] = [];
|
||||
const system = `You are an expert nursing educator and item writer reviewing existing exam questions.
|
||||
Improve each question while keeping what it tests and its correct answer(s) unless the keyed answer is factually wrong (if so, fix it and say so at the start of the rationale).
|
||||
- Make the stem clear, complete and free of clues; fix grammar.
|
||||
- Replace weak or implausible distractors; keep choices parallel and similar in length; remove "all/none of the above".
|
||||
- Write a teaching rationale, and give every choice a one-sentence "why".
|
||||
- Keep the same question type unless it is clearly wrong for the content. Set a short topic label and a difficulty.
|
||||
Return the questions in the same order, one output question per input question.${req.instructions ? `\n\nAdditional instructions: ${req.instructions}` : ''}`;
|
||||
for (let i = 0; i < questions.length; i += 8) {
|
||||
if (controller.signal.aborted) throw new AIError('aborted', 'Cancelled.');
|
||||
const batch = questions.slice(i, i + 8);
|
||||
setProgress(app.db, draftId, 0.05 + (0.9 * i) / questions.length, `Reviewing questions ${i + 1}-${i + batch.length} of ${questions.length}`);
|
||||
const body = batch
|
||||
.map(
|
||||
(q, n) =>
|
||||
`Question ${n + 1} (${q.type})\nStem: ${q.stem}\nChoices:\n${q.options.map((o) => `- [${o.correct ? 'x' : ' '}] ${o.text}`).join('\n')}\nRationale: ${q.rationale || '(none)'}\nTopic: ${q.topic || '(none)'}`,
|
||||
)
|
||||
.join('\n\n');
|
||||
const result = await generateStructured(app, {
|
||||
task: 'generate',
|
||||
purpose: 'improve questions',
|
||||
modelId: req.modelId,
|
||||
system,
|
||||
messages: [{ role: 'user', content: [{ type: 'text', text: body }] }],
|
||||
toolName: 'submit_questions',
|
||||
toolDescription: 'Submit the improved questions in the original order.',
|
||||
schema: QUESTION_SCHEMA,
|
||||
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === 'quiz') as QuizDraft | undefined;
|
||||
const improved = parsed?.questions ?? [];
|
||||
batch.forEach((orig, n) => {
|
||||
const next = improved[n];
|
||||
if (!next || validateDraftQuestion(next, 0).some((p) => p.level === 'error')) return;
|
||||
next.source = undefined;
|
||||
out.push(next);
|
||||
ids.push(orig.id);
|
||||
});
|
||||
}
|
||||
if (out.length === 0) throw new AIError('invalid_response', 'The model did not return usable revisions. Try again or use a different model.');
|
||||
const payload: QuizDraft = { kind: 'quiz', title: quiz.title, description: quiz.description, questions: out };
|
||||
app.db.run(
|
||||
`UPDATE drafts SET status = 'ready', payload = ?, meta = ?, progress = 1, progress_label = '', updated_at = ? WHERE id = ?`,
|
||||
JSON.stringify(payload),
|
||||
JSON.stringify({ origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: ids } satisfies DraftMeta),
|
||||
now(),
|
||||
draftId,
|
||||
);
|
||||
})()
|
||||
.catch((err) => {
|
||||
const message = err instanceof AIError || (err instanceof Error && 'status' in err) ? (err as Error).message : 'Improving the questions failed unexpectedly.';
|
||||
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId);
|
||||
})
|
||||
.then(() => {
|
||||
running.delete(draftId);
|
||||
return getDraft(app.db, draftId);
|
||||
});
|
||||
return { draftId, done };
|
||||
}
|
||||
581
server/ai/providers/anthropic.ts
Normal file
581
server/ai/providers/anthropic.ts
Normal file
|
|
@ -0,0 +1,581 @@
|
|||
/**
|
||||
* Anthropic — Messages API (`POST /v1/messages`, SSE).
|
||||
*
|
||||
* Verified against (2026-11):
|
||||
* https://platform.claude.com/docs/en/api/messages
|
||||
* https://platform.claude.com/docs/en/build-with-claude/streaming
|
||||
* https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
|
||||
* https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
|
||||
* https://platform.claude.com/docs/en/api/migration-2026-10-21
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - API version 2026-10-21 rejects temperature/top_p/top_k on EVERY model, so
|
||||
* this adapter never sends a temperature (the catalog marks Claude models
|
||||
* `temperature: false`).
|
||||
* - `max_tokens` is required; thinking counts against it.
|
||||
* - Thinking configuration differs per model generation (see `claudeProfile`).
|
||||
* - Thinking blocks (with their signatures) must be replayed unchanged, so the
|
||||
* raw content blocks of each turn are kept in `providerData`.
|
||||
* - Newer models bind thinking blocks to the conversation prefix (system
|
||||
* prompt, tool set, earlier messages). If the app changed any of those, the
|
||||
* API answers 400 "Invalid `signature` in `thinking` block"; the documented
|
||||
* recovery is to strip the thinking blocks and retry once.
|
||||
* - Safety classifiers can end a turn with `stop_reason: "refusal"` (HTTP 200).
|
||||
* For the models that support it we opt into server-side fallback, which
|
||||
* re-runs a declined request on another Claude model inside the same call.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'Anthropic';
|
||||
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
|
||||
const API_VERSION = '2026-10-21';
|
||||
/** Beta that enables the `fallbacks` request parameter. */
|
||||
const FALLBACK_BETA = 'server-side-fallback-2026-07-01';
|
||||
/** Streaming has no HTTP-timeout concern, so give the model room (thinking counts against this). */
|
||||
const DEFAULT_MAX_TOKENS = 64_000;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-model behaviour */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface ClaudeProfile {
|
||||
/**
|
||||
* always – thinking cannot be turned off (Fable/Mythos, Opus 5.5+)
|
||||
* default_on – adaptive thinking runs unless disabled (Opus 5, Sonnet 5+, Haiku 5.5)
|
||||
* opt_in – adaptive thinking must be requested (Opus 4.6–4.8, Sonnet 4.6)
|
||||
* budget – legacy `budget_tokens` thinking (Haiku 4.5, Opus/Sonnet 4.5 and older)
|
||||
* none – no thinking support (Claude 3 and earlier)
|
||||
*/
|
||||
thinking: 'always' | 'default_on' | 'opt_in' | 'budget' | 'none';
|
||||
/** How thinking is switched off, when the model allows it. */
|
||||
off: 'disabled' | 'between_tools' | null;
|
||||
/** Accepts `output_config.effort`. */
|
||||
effort: boolean;
|
||||
/** Accepts `thinking.display` (and defaults to hiding the reasoning text). */
|
||||
display: boolean;
|
||||
/** Accepts `tool_choice` of type `any` / `tool`. */
|
||||
forcedToolChoice: boolean;
|
||||
/** Supports the server-side refusal `fallbacks` parameter. */
|
||||
fallbacks: boolean;
|
||||
}
|
||||
|
||||
/** Thinking / tool-choice rules by model, from the model migration guides. */
|
||||
export function claudeProfile(model: string): ClaudeProfile {
|
||||
const m = /claude-(fable|mythos|opus|sonnet|haiku)-(\d+)(?:-(\d{1,2}))?(?!\d)/.exec(model);
|
||||
if (!m) {
|
||||
// "claude-3-5-sonnet-…" and older put the version first and never had adaptive thinking.
|
||||
if (/claude-(?:[123]\b|instant)/.test(model)) {
|
||||
return { thinking: 'none', off: null, effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
// Unknown naming scheme: assume it behaves like the newest generation.
|
||||
return { thinking: 'default_on', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: false };
|
||||
}
|
||||
const family = m[1];
|
||||
const v = Number(m[2]) * 10 + Number(m[3] ?? 0); // 4.6 -> 46, 5 -> 50, 5.5 -> 55
|
||||
|
||||
if (family === 'fable' || family === 'mythos') {
|
||||
return {
|
||||
thinking: 'always',
|
||||
off: null,
|
||||
effort: true,
|
||||
display: true,
|
||||
forcedToolChoice: v <= 50,
|
||||
fallbacks: family === 'fable' && (v === 51 || v === 52),
|
||||
};
|
||||
}
|
||||
if (family === 'opus') {
|
||||
if (v >= 55) return { thinking: 'always', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: v === 50 };
|
||||
if (v >= 47) return { thinking: 'opt_in', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: v === 45, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
if (family === 'sonnet') {
|
||||
// Sonnet 5.5 rejects {type:"disabled"}; its lowest setting is {type:"between_tools"}.
|
||||
if (v >= 55) return { thinking: 'default_on', off: 'between_tools', effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
|
||||
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
// haiku
|
||||
if (v >= 55) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
|
||||
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
|
||||
}
|
||||
|
||||
/** Thinking budgets for the legacy `budget_tokens` models. */
|
||||
const LEGACY_BUDGET = { low: 2_048, medium: 8_192, high: 16_384 } as const;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Request building */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
interface BuildOptions {
|
||||
/** Official endpoint: enables features a proxy may not understand. */
|
||||
official: boolean;
|
||||
stripThinking: boolean;
|
||||
useFallbacks: boolean;
|
||||
eagerToolInput: boolean;
|
||||
promptCache: boolean;
|
||||
}
|
||||
|
||||
/** Tool-call ids must match ^[a-zA-Z0-9_-]+$; ids minted by other providers may not. */
|
||||
function safeId(id: string): string {
|
||||
const cleaned = id.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return cleaned || 'tool_call';
|
||||
}
|
||||
|
||||
function userBlocks(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const results: Json[] = [];
|
||||
const rest: Json[] = [];
|
||||
for (const p of parts) {
|
||||
switch (p.type) {
|
||||
case 'tool_result': {
|
||||
// All results of one assistant turn go into ONE user message, before any other content.
|
||||
const block: Json = { type: 'tool_result', tool_use_id: safeId(p.toolCallId), content: p.output || '(no output)' };
|
||||
if (p.isError) block.is_error = true;
|
||||
results.push(block);
|
||||
break;
|
||||
}
|
||||
case 'text':
|
||||
if (p.text) rest.push({ type: 'text', text: p.text });
|
||||
break;
|
||||
case 'image':
|
||||
if (caps.vision === false) rest.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else rest.push({ type: 'image', source: { type: 'base64', media_type: p.mime, data: p.data } });
|
||||
break;
|
||||
case 'file':
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) {
|
||||
const doc: Json = { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: p.data } };
|
||||
if (p.filename) doc.title = p.filename;
|
||||
rest.push(doc);
|
||||
} else if (p.mime.startsWith('text/')) {
|
||||
const doc: Json = { type: 'document', source: { type: 'text', media_type: 'text/plain', data: Buffer.from(p.data, 'base64').toString('utf8') } };
|
||||
if (p.filename) doc.title = p.filename;
|
||||
rest.push(doc);
|
||||
} else {
|
||||
rest.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
break;
|
||||
case 'tool_call':
|
||||
break; // not valid in a user message
|
||||
}
|
||||
}
|
||||
return [...results, ...rest];
|
||||
}
|
||||
|
||||
function assistantBlocks(msg: ChatMessage, stripThinking: boolean): Json[] {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'anthropic' && isRecord(pd.data) && Array.isArray(pd.data.content)) {
|
||||
// Same provider: replay the turn exactly as it was received, thinking blocks
|
||||
// and signatures included.
|
||||
const blocks = pd.data.content.filter(isRecord);
|
||||
return stripThinking ? blocks.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking') : blocks;
|
||||
}
|
||||
// Another provider produced this turn: rebuild it from the neutral parts.
|
||||
const blocks: Json[] = [];
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
||||
else if (p.type === 'tool_call') blocks.push({ type: 'tool_use', id: safeId(p.id), name: p.name, input: p.input });
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function toWireMessages(messages: ChatMessage[], caps: Partial<ModelCapabilities>, stripThinking: boolean): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
const content = msg.role === 'user' ? userBlocks(msg.content, caps) : assistantBlocks(msg, stripThinking);
|
||||
if (content.length === 0) continue; // the API rejects empty turns
|
||||
out.push({ role: msg.role, content });
|
||||
}
|
||||
if (out[0]?.role === 'assistant') out.unshift({ role: 'user', content: [{ type: 'text', text: '(continuing the conversation)' }] });
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, profile: ClaudeProfile, opts: BuildOptions): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const defaults = lookupModelDefaults('anthropic', req.model);
|
||||
const ceiling = defaults.maxOutput ?? Number.POSITIVE_INFINITY;
|
||||
const maxTokens = Math.max(1, Math.min(req.maxOutputTokens ?? DEFAULT_MAX_TOKENS, ceiling));
|
||||
|
||||
const systemNotes: string[] = [];
|
||||
const body: Json = { model: req.model, max_tokens: maxTokens, stream: true };
|
||||
|
||||
// --- tools -------------------------------------------------------
|
||||
const tools = caps.tools === false ? [] : (req.tools ?? []);
|
||||
let forcing = false;
|
||||
if (tools.length > 0) {
|
||||
body.tools = tools.map((t) => {
|
||||
const tool: Json = { name: t.name, description: t.description, input_schema: { type: 'object', ...t.parameters } };
|
||||
// Without this the API buffers each tool-input parameter and the stream goes
|
||||
// silent for the whole time a large input (a generated quiz, say) is written.
|
||||
// The price is that the input is no longer validated server-side, so it is
|
||||
// parsed strictly below and truncated calls are dropped.
|
||||
if (opts.eagerToolInput) tool.eager_input_streaming = true;
|
||||
return tool;
|
||||
});
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none') {
|
||||
body.tool_choice = { type: 'none' };
|
||||
} else if (choice === 'required' || isRecord(choice)) {
|
||||
const name = isRecord(choice) ? str(choice.name) : '';
|
||||
if (profile.forcedToolChoice) {
|
||||
body.tool_choice = name ? { type: 'tool', name } : { type: 'any' };
|
||||
forcing = true;
|
||||
} else {
|
||||
// Fable 5.1+, Opus 5.5 and Sonnet 5.5 answer 400 to forced tool use; the
|
||||
// documented replacement is `auto` plus an explicit instruction.
|
||||
systemNotes.push(
|
||||
name
|
||||
? `For this turn you must call the \`${name}\` tool. Begin your response with that tool call.`
|
||||
: 'For this turn you must call one of the available tools. Begin your response with the tool call.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- thinking ----------------------------------------------------
|
||||
if (caps.reasoning !== false && profile.thinking !== 'none') {
|
||||
const effort = req.reasoningEffort;
|
||||
const adaptive: Json = profile.display ? { type: 'adaptive', display: 'summarized' } : { type: 'adaptive' };
|
||||
const outputConfig: Json = {};
|
||||
if (effort === 'none') {
|
||||
if (profile.thinking === 'always') {
|
||||
// Cannot be disabled: the lowest effort is the closest thing to "off".
|
||||
body.thinking = adaptive;
|
||||
outputConfig.effort = 'low';
|
||||
} else if (profile.thinking === 'default_on') {
|
||||
body.thinking = { type: profile.off ?? 'disabled' }; // `between_tools` takes no other field
|
||||
}
|
||||
// opt_in / budget models do not think unless asked.
|
||||
} else if (profile.thinking === 'always' || profile.thinking === 'default_on') {
|
||||
// `display: "summarized"` because these models hide the reasoning text by
|
||||
// default, which would look like a long pause in the UI.
|
||||
body.thinking = adaptive;
|
||||
if (effort) outputConfig.effort = effort;
|
||||
} else if (effort) {
|
||||
// Forced tool use cannot be combined with thinking on the older generations.
|
||||
if (profile.thinking === 'opt_in' && !forcing) body.thinking = adaptive;
|
||||
if (profile.thinking === 'budget' && !forcing) {
|
||||
const budget = Math.min(LEGACY_BUDGET[effort], maxTokens - 1_024); // must be < max_tokens, min 1024
|
||||
if (budget >= 1_024) body.thinking = { type: 'enabled', budget_tokens: budget };
|
||||
}
|
||||
if (profile.effort) outputConfig.effort = effort;
|
||||
}
|
||||
if (Object.keys(outputConfig).length > 0) body.output_config = outputConfig;
|
||||
}
|
||||
|
||||
// --- system prompt -----------------------------------------------
|
||||
// There is no schema-less JSON mode (structured outputs need a schema), and
|
||||
// assistant prefill is rejected by current models, so JSON mode is an instruction.
|
||||
if (req.jsonMode) systemNotes.push('Respond with a single valid JSON object and nothing else — no prose, no Markdown code fences.');
|
||||
const system = [req.system?.trim(), ...systemNotes].filter(Boolean).join('\n\n');
|
||||
if (system) body.system = system;
|
||||
|
||||
body.messages = toWireMessages(req.messages, caps, opts.stripThinking);
|
||||
|
||||
if (opts.promptCache) body.cache_control = { type: 'ephemeral' }; // automatic caching of the reusable prefix
|
||||
if (opts.useFallbacks) body.fallbacks = 'default';
|
||||
return body;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Adapter */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function rootUrl(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1$/, '');
|
||||
}
|
||||
|
||||
function isOfficial(cfg: ProviderConfig): boolean {
|
||||
try {
|
||||
return new URL(rootUrl(cfg)).hostname === 'api.anthropic.com';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig, betas: string[] = []): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an Anthropic API key in Settings to use Claude models.');
|
||||
const h: Record<string, string> = { ...cfg.headers, 'x-api-key': cfg.apiKey, 'anthropic-version': API_VERSION };
|
||||
const userBeta = Object.entries(cfg.headers ?? {}).find(([k]) => k.toLowerCase() === 'anthropic-beta')?.[1];
|
||||
const all = [...(userBeta ? userBeta.split(',').map((s) => s.trim()) : []), ...betas].filter(Boolean);
|
||||
for (const k of Object.keys(h)) if (k.toLowerCase() === 'anthropic-beta') delete h[k];
|
||||
if (all.length > 0) h['anthropic-beta'] = [...new Set(all)].join(',');
|
||||
return h;
|
||||
}
|
||||
|
||||
function mapStopReason(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'end_turn':
|
||||
case 'stop_sequence':
|
||||
return 'end';
|
||||
case 'tool_use':
|
||||
return 'tool_use';
|
||||
case 'max_tokens':
|
||||
case 'model_context_window_exceeded':
|
||||
return 'max_tokens';
|
||||
case 'refusal':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other'; // pause_turn (server tools only) and anything new
|
||||
}
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const profile = claudeProfile(req.model);
|
||||
const official = isOfficial(cfg);
|
||||
const opts: BuildOptions = {
|
||||
official,
|
||||
stripThinking: false,
|
||||
// The beta-gated extras are only sent to api.anthropic.com; gateways and
|
||||
// proxies frequently reject fields they do not know.
|
||||
useFallbacks: official && profile.fallbacks,
|
||||
eagerToolInput: official,
|
||||
promptCache: official,
|
||||
};
|
||||
|
||||
// Open the stream, degrading once per recognised 400 (see file header).
|
||||
let http;
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
http = await httpRequest(`${rootUrl(cfg)}/v1/messages`, {
|
||||
...ctx,
|
||||
headers: headers(cfg, opts.useFallbacks ? [FALLBACK_BETA] : []),
|
||||
body: buildBody(req, profile, opts),
|
||||
signal: req.signal,
|
||||
});
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
||||
const detail = err.providerMessage ?? '';
|
||||
if (!opts.stripThinking && /signature/i.test(detail) && /thinking/i.test(detail)) opts.stripThinking = true;
|
||||
else if (opts.useFallbacks && /fallback|anthropic-beta/i.test(detail)) opts.useFallbacks = false;
|
||||
else if (opts.eagerToolInput && /eager_input_streaming/i.test(detail)) opts.eagerToolInput = false;
|
||||
else if (opts.promptCache && /cache_control/i.test(detail)) opts.promptCache = false;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Content blocks by index, accumulated exactly as the API defines them.
|
||||
const blocks: Json[] = [];
|
||||
const toolJson = new Map<number, string>();
|
||||
let stopReason = '';
|
||||
let sawStop = false;
|
||||
let reasoningEmitted = false;
|
||||
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
|
||||
const readUsage = (u: unknown) => {
|
||||
if (!isRecord(u)) return;
|
||||
if (typeof u.input_tokens === 'number') usage.input = u.input_tokens;
|
||||
if (typeof u.cache_read_input_tokens === 'number') usage.cacheRead = u.cache_read_input_tokens;
|
||||
if (typeof u.cache_creation_input_tokens === 'number') usage.cacheWrite = u.cache_creation_input_tokens;
|
||||
if (typeof u.output_tokens === 'number') usage.output = u.output_tokens;
|
||||
};
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
switch (data.type) {
|
||||
case 'message_start':
|
||||
if (isRecord(data.message)) readUsage(data.message.usage);
|
||||
break;
|
||||
case 'content_block_start': {
|
||||
const index = num(data.index);
|
||||
const block: Json = isRecord(data.content_block) ? { ...data.content_block } : { type: 'unknown' };
|
||||
blocks[index] = block;
|
||||
if (block.type === 'tool_use') toolJson.set(index, '');
|
||||
if (block.type === 'text' && typeof block.text === 'string' && block.text) yield { type: 'text', delta: block.text };
|
||||
if (block.type === 'thinking' && reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
||||
break;
|
||||
}
|
||||
case 'content_block_delta': {
|
||||
const index = num(data.index);
|
||||
const block = blocks[index];
|
||||
const delta = data.delta;
|
||||
if (!block || !isRecord(delta)) break;
|
||||
if (delta.type === 'text_delta') {
|
||||
const text = str(delta.text);
|
||||
block.text = str(block.text) + text;
|
||||
if (text) yield { type: 'text', delta: text };
|
||||
} else if (delta.type === 'thinking_delta') {
|
||||
const text = str(delta.thinking);
|
||||
block.thinking = str(block.thinking) + text;
|
||||
if (text) {
|
||||
reasoningEmitted = true;
|
||||
yield { type: 'reasoning', delta: text };
|
||||
}
|
||||
} else if (delta.type === 'signature_delta') {
|
||||
block.signature = str(block.signature) + str(delta.signature);
|
||||
} else if (delta.type === 'input_json_delta') {
|
||||
toolJson.set(index, (toolJson.get(index) ?? '') + str(delta.partial_json));
|
||||
} else if (delta.type === 'citations_delta') {
|
||||
const list = Array.isArray(block.citations) ? block.citations : [];
|
||||
list.push(delta.citation);
|
||||
block.citations = list;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'message_delta':
|
||||
if (isRecord(data.delta) && typeof data.delta.stop_reason === 'string') stopReason = data.delta.stop_reason;
|
||||
readUsage(data.usage);
|
||||
break;
|
||||
case 'message_stop':
|
||||
sawStop = true;
|
||||
break;
|
||||
case 'error':
|
||||
// e.g. overloaded_error delivered after the 200 response started
|
||||
throw errorFromProvider(ctx, undefined, data);
|
||||
default:
|
||||
break; // ping, and event types added later
|
||||
}
|
||||
}
|
||||
if (!sawStop && !stopReason) {
|
||||
throw new AIError('network', 'The connection to Anthropic closed before the response finished.', { retryable: true });
|
||||
}
|
||||
|
||||
// Tool calls are surfaced only now, at the end of the message, because the
|
||||
// stop reason decides whether they are usable:
|
||||
// - `refusal` can cut a tool_use off mid-input — never run that turn's tools;
|
||||
// - `max_tokens` can truncate the last call — drop it rather than "repair" it;
|
||||
// - after a mid-output fallback, tool_use blocks from before the last
|
||||
// `fallback` block belong to the declined attempt and must not be echoed.
|
||||
let boundary = -1;
|
||||
blocks.forEach((b, i) => {
|
||||
if (b?.type === 'fallback') boundary = i;
|
||||
});
|
||||
|
||||
const content: ContentPart[] = [];
|
||||
const replay: Json[] = [];
|
||||
const calls: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
if (!block) continue;
|
||||
const beforeBoundary = i < boundary;
|
||||
if (block.type === 'text') {
|
||||
const text = str(block.text);
|
||||
if (!text) continue; // empty text blocks are rejected when echoed back
|
||||
content.push({ type: 'text', text });
|
||||
replay.push(block);
|
||||
} else if (block.type === 'tool_use') {
|
||||
if (beforeBoundary || stopReason === 'refusal') continue;
|
||||
const raw = toolJson.get(i) ?? '';
|
||||
const name = str(block.name);
|
||||
let input = parseArgsStrict(raw);
|
||||
if (!input) {
|
||||
if (stopReason === 'max_tokens') continue; // truncated mid-input
|
||||
input = parseToolArguments(raw, LABEL, name);
|
||||
}
|
||||
const call = { id: str(block.id), name, input };
|
||||
calls.push(call);
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
replay.push({ ...block, input });
|
||||
} else if (block.type === 'fallback') {
|
||||
replay.push(block); // must stay exactly where it appeared
|
||||
} else if (!beforeBoundary) {
|
||||
replay.push(block); // thinking, redacted_thinking, …: echoed back unchanged
|
||||
}
|
||||
}
|
||||
|
||||
for (const call of calls) yield { type: 'tool_call', call };
|
||||
|
||||
let mapped = mapStopReason(stopReason);
|
||||
if (mapped === 'tool_use' && calls.length === 0) mapped = 'other';
|
||||
if (mapped === 'end' && calls.length > 0) mapped = 'tool_use';
|
||||
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
if (replay.length > 0) message.providerData = { kind: 'anthropic', data: { content: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: mapped,
|
||||
// Anthropic reports uncached, cache-read and cache-write input separately.
|
||||
usage: makeUsage({
|
||||
input: usage.input + usage.cacheRead + usage.cacheWrite,
|
||||
output: usage.output,
|
||||
cacheRead: usage.cacheRead,
|
||||
cacheWrite: usage.cacheWrite,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function supported(caps: unknown, key: string): boolean | undefined {
|
||||
if (!isRecord(caps) || !isRecord(caps[key])) return undefined;
|
||||
const v = (caps[key] as Json).supported;
|
||||
return typeof v === 'boolean' ? v : undefined;
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const ctx = errorContext(cfg);
|
||||
const found: DiscoveredModel[] = [];
|
||||
let afterId = '';
|
||||
// https://platform.claude.com/docs/en/api/models/list — cursor pagination via has_more / last_id.
|
||||
for (let page = 0; page < 20; page++) {
|
||||
const url = `${rootUrl(cfg)}/v1/models?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ''}`;
|
||||
const res = await requestJson<Json>(url, { ...ctx, headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
||||
for (const m of data) {
|
||||
const id = str(m.id);
|
||||
if (!id || !isChatModelId(id)) continue;
|
||||
const model: DiscoveredModel = {
|
||||
id,
|
||||
// The API version in use rejects sampling parameters on every model.
|
||||
capabilities: { tools: true, temperature: false },
|
||||
};
|
||||
if (typeof m.display_name === 'string') model.label = m.display_name;
|
||||
if (typeof m.max_input_tokens === 'number') model.contextWindow = m.max_input_tokens;
|
||||
if (typeof m.max_tokens === 'number') model.maxOutput = m.max_tokens;
|
||||
const vision = supported(m.capabilities, 'image_input');
|
||||
const pdf = supported(m.capabilities, 'pdf_input');
|
||||
const thinking = supported(m.capabilities, 'thinking');
|
||||
if (vision !== undefined) model.capabilities = { ...model.capabilities, vision };
|
||||
if (pdf !== undefined) model.capabilities = { ...model.capabilities, pdf };
|
||||
if (thinking !== undefined) model.capabilities = { ...model.capabilities, reasoning: thinking };
|
||||
found.push(model);
|
||||
}
|
||||
if (res.has_more !== true || typeof res.last_id !== 'string' || !res.last_id) break;
|
||||
afterId = res.last_id;
|
||||
}
|
||||
return withCatalogDefaults('anthropic', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
// Listing models is free and proves both connectivity and the key.
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const anthropicAdapter: ProviderAdapter = {
|
||||
kind: 'anthropic',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://platform.claude.com/settings/keys',
|
||||
keyHint: 'sk-ant-…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
206
server/ai/providers/catalog.ts
Normal file
206
server/ai/providers/catalog.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
/**
|
||||
* Static defaults for well-known models: capabilities, limits and list prices.
|
||||
*
|
||||
* Discovery APIs return ids (and sometimes limits) but almost never prices or
|
||||
* feature flags, so `listModels` merges its results with this table. Unknown
|
||||
* models simply get no defaults — nothing here is required for a model to work.
|
||||
*
|
||||
* Everything below was read from the official pages listed per provider on the
|
||||
* dates noted. Prices are USD per million tokens, standard (non-batch,
|
||||
* non-priority) tier, short-context tier where a provider has several. A field
|
||||
* is omitted when it could not be confirmed from the page — do not guess.
|
||||
*/
|
||||
import type { DiscoveredModel, ModelCapabilities, ProviderKind } from '../types.ts';
|
||||
|
||||
export const CATALOG_UPDATED = '2026-11-10';
|
||||
|
||||
type Entry = Omit<DiscoveredModel, 'id' | 'capabilities'> & { capabilities?: Partial<ModelCapabilities> };
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI */
|
||||
/* Prices: https://developers.openai.com/api/docs/pricing (fetched 2026-11-11)
|
||||
/* Limits: https://developers.openai.com/api/docs/models (fetched 2026-11-11;
|
||||
/* only the four current flagship models list limits there — the
|
||||
/* limits on older models below are the long-published values and were
|
||||
/* not re-fetched)
|
||||
/* Reasoning models reject a custom `temperature`, hence temperature:false.
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const OA_REASONING: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
||||
const OA_CLASSIC: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: false, temperature: true };
|
||||
const OA_FLAGSHIP_LIMITS = { contextWindow: 1_050_000, maxOutput: 128_000 };
|
||||
|
||||
const OPENAI: Record<string, Entry> = {
|
||||
'gpt-6-astra': { label: 'GPT-6 Astra', ...OA_FLAGSHIP_LIMITS, priceIn: 10, priceOut: 50, capabilities: OA_REASONING },
|
||||
'gpt-6.1-sol': { label: 'GPT-6.1 Sol', ...OA_FLAGSHIP_LIMITS, priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-6.1-terra': { label: 'GPT-6.1 Terra', ...OA_FLAGSHIP_LIMITS, priceIn: 1, priceOut: 5, capabilities: OA_REASONING },
|
||||
'gpt-6-luna': { label: 'GPT-6 Luna', ...OA_FLAGSHIP_LIMITS, priceIn: 0.1, priceOut: 0.5, capabilities: OA_REASONING },
|
||||
'gpt-6-sol': { label: 'GPT-6 Sol', priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5.6-sol': { label: 'GPT-5.6 Sol', priceIn: 4, priceOut: 20, capabilities: OA_REASONING },
|
||||
'gpt-5.6-terra': { label: 'GPT-5.6 Terra', priceIn: 2, priceOut: 12, capabilities: OA_REASONING },
|
||||
'gpt-5.6-luna': { label: 'GPT-5.6 Luna', priceIn: 0.2, priceOut: 1.2, capabilities: OA_REASONING },
|
||||
'gpt-5.5': { label: 'GPT-5.5', priceIn: 5, priceOut: 30, capabilities: OA_REASONING },
|
||||
'gpt-5.5-pro': { label: 'GPT-5.5 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
||||
'gpt-5.4': { label: 'GPT-5.4', priceIn: 2.5, priceOut: 15, capabilities: OA_REASONING },
|
||||
'gpt-5.4-pro': { label: 'GPT-5.4 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
|
||||
'gpt-5.4-mini': { label: 'GPT-5.4 mini', priceIn: 0.75, priceOut: 4.5, capabilities: OA_REASONING },
|
||||
'gpt-5.4-nano': { label: 'GPT-5.4 nano', priceIn: 0.2, priceOut: 1.25, capabilities: OA_REASONING },
|
||||
'gpt-5.3-codex': { label: 'GPT-5.3 Codex', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
||||
'gpt-5.2': { label: 'GPT-5.2', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
|
||||
'gpt-5.2-pro': { label: 'GPT-5.2 Pro', priceIn: 21, priceOut: 168, capabilities: OA_REASONING },
|
||||
'gpt-5.1': { label: 'GPT-5.1', priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5': { label: 'GPT-5', contextWindow: 400_000, maxOutput: 128_000, priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
|
||||
'gpt-5-mini': { label: 'GPT-5 mini', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.25, priceOut: 2, capabilities: OA_REASONING },
|
||||
'gpt-5-nano': { label: 'GPT-5 nano', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.05, priceOut: 0.4, capabilities: OA_REASONING },
|
||||
'gpt-5-pro': { label: 'GPT-5 Pro', priceIn: 15, priceOut: 120, capabilities: OA_REASONING },
|
||||
'chat-latest': { label: 'ChatGPT (latest)', priceIn: 5, priceOut: 30, capabilities: OA_CLASSIC },
|
||||
'gpt-4.1': { label: 'GPT-4.1', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 2, priceOut: 8, capabilities: OA_CLASSIC },
|
||||
'gpt-4.1-mini': { label: 'GPT-4.1 mini', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 0.4, priceOut: 1.6, capabilities: OA_CLASSIC },
|
||||
'gpt-4o': { label: 'GPT-4o', contextWindow: 128_000, maxOutput: 16_384, priceIn: 2.5, priceOut: 10, capabilities: OA_CLASSIC },
|
||||
'gpt-4o-mini': { label: 'GPT-4o mini', contextWindow: 128_000, maxOutput: 16_384, priceIn: 0.15, priceOut: 0.6, capabilities: OA_CLASSIC },
|
||||
o3: { label: 'o3', contextWindow: 200_000, maxOutput: 100_000, priceIn: 2, priceOut: 8, capabilities: OA_REASONING },
|
||||
'o3-pro': { label: 'o3-pro', contextWindow: 200_000, maxOutput: 100_000, priceIn: 20, priceOut: 80, capabilities: OA_REASONING },
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Anthropic */
|
||||
/* Prices: https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-11-11)
|
||||
/* Ids and limits: https://platform.claude.com/docs/en/about-claude/models/overview
|
||||
/* (via Anthropic's Claude API reference, cached 2026-11-03)
|
||||
/* temperature:false for every Claude model: the adapter speaks API version
|
||||
/* 2026-10-21, which rejects temperature/top_p/top_k on all models
|
||||
/* (https://platform.claude.com/docs/en/api/migration-2026-10-21).
|
||||
/* Claude Haiku 5.5 prices are for prompts up to 100K tokens ($0.50/$2.50 above).
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const CLAUDE: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
|
||||
const CLAUDE_1M = { contextWindow: 1_000_000, maxOutput: 128_000 };
|
||||
|
||||
const ANTHROPIC: Record<string, Entry> = {
|
||||
'claude-fable-5-2': { label: 'Claude Fable 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
||||
'claude-mythos-5-2': { label: 'Claude Mythos 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
|
||||
'claude-fable-5-1': { label: 'Claude Fable 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-mythos-5-1': { label: 'Claude Mythos 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-fable-5': { label: 'Claude Fable 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-mythos-5': { label: 'Claude Mythos 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
|
||||
'claude-opus-5-5': { label: 'Claude Opus 5.5', ...CLAUDE_1M, priceIn: 4, priceOut: 20, capabilities: CLAUDE },
|
||||
'claude-opus-5': { label: 'Claude Opus 5', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-8': { label: 'Claude Opus 4.8', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-7': { label: 'Claude Opus 4.7', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-6': { label: 'Claude Opus 4.6', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-opus-4-5': { label: 'Claude Opus 4.5', priceIn: 5, priceOut: 25, capabilities: CLAUDE },
|
||||
'claude-sonnet-5-5': { label: 'Claude Sonnet 5.5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
||||
'claude-sonnet-5': { label: 'Claude Sonnet 5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
|
||||
'claude-sonnet-4-6': { label: 'Claude Sonnet 4.6', ...CLAUDE_1M, priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
||||
'claude-sonnet-4-5': { label: 'Claude Sonnet 4.5', priceIn: 3, priceOut: 15, capabilities: CLAUDE },
|
||||
'claude-haiku-5-5': { label: 'Claude Haiku 5.5', ...CLAUDE_1M, priceIn: 0.1, priceOut: 0.5, capabilities: CLAUDE },
|
||||
'claude-haiku-4-5': { label: 'Claude Haiku 4.5', contextWindow: 200_000, maxOutput: 64_000, priceIn: 1, priceOut: 5, capabilities: CLAUDE },
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Google Gemini */
|
||||
/* Prices: https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-11-11,
|
||||
/* now served from https://aistudio.google.com/docs/pricing)
|
||||
/* Limits: https://ai.google.dev/gemini-api/docs/models (fetched 2026-11-11;
|
||||
/* only the Gemini 4 models list limits — `GET /v1beta/models`
|
||||
/* supplies inputTokenLimit/outputTokenLimit for the rest at runtime)
|
||||
/* Scheduled changes announced on the pricing page (NOT applied here):
|
||||
/* - gemini-4-carbon / gemini-4-argon: $4 / $20 from 2026-12-01
|
||||
/* - gemini-3.8-flash / gemini-3.6-flash: $1.50 / $7.50 from 2027-01-01
|
||||
/* - gemini-2.5-pro / gemini-2.5-flash are deprecated and are routed to
|
||||
/* gemini-3.8-flash / gemini-3.5-flash-lite (and billed as such) from 2026-12-01
|
||||
/* Pro-tier prices are for prompts up to 200K tokens.
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const GEMINI_CAPS: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: true };
|
||||
const GEMINI_4_LIMITS = { contextWindow: 1_048_576, maxOutput: 65_536 };
|
||||
|
||||
const GEMINI: Record<string, Entry> = {
|
||||
'gemini-4-carbon': { label: 'Gemini 4 Carbon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-4-argon': { label: 'Gemini 4 Argon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.8-flash': { label: 'Gemini 3.8 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.6-flash': { label: 'Gemini 3.6 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.5-flash-lite': { label: 'Gemini 3.5 Flash-Lite', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.1-pro-preview': { label: 'Gemini 3.1 Pro (preview)', priceIn: 2, priceOut: 12, capabilities: GEMINI_CAPS },
|
||||
'gemini-3.1-flash-lite': { label: 'Gemini 3.1 Flash-Lite', priceIn: 0.25, priceOut: 1.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-3-flash-preview': { label: 'Gemini 3 Flash (preview)', priceIn: 0.5, priceOut: 3, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-pro': { label: 'Gemini 2.5 Pro', priceIn: 1.25, priceOut: 10, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-flash': { label: 'Gemini 2.5 Flash', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
|
||||
'gemini-2.5-flash-lite': { label: 'Gemini 2.5 Flash-Lite', priceIn: 0.1, priceOut: 0.4, capabilities: GEMINI_CAPS },
|
||||
};
|
||||
|
||||
const TABLES: Record<ProviderKind, Record<string, Entry>> = {
|
||||
openai: OPENAI,
|
||||
anthropic: ANTHROPIC,
|
||||
gemini: GEMINI,
|
||||
// Aggregators and local servers host arbitrary models at their own prices.
|
||||
openai_compatible: {},
|
||||
ollama: {},
|
||||
};
|
||||
|
||||
/**
|
||||
* What may follow a known id for it to still be "the same model at the same
|
||||
* price": a dated snapshot or a moving alias, e.g. `gpt-5-2025-08-07`,
|
||||
* `claude-haiku-4-5-20251001`, `claude-opus-4-5@20251101`, `gemini-2.5-flash-latest`.
|
||||
*/
|
||||
const SNAPSHOT_SUFFIX = /^[-@:](\d{4}-\d{2}-\d{2}|\d{6,8}|\d{3,4}|latest|preview|preview-\d{2}-\d{2,4}|v\d+(:\d+)?)$/;
|
||||
|
||||
function clone(e: Entry): Partial<DiscoveredModel> {
|
||||
return { ...e, ...(e.capabilities ? { capabilities: { ...e.capabilities } } : {}) };
|
||||
}
|
||||
|
||||
function lookupIn(table: Record<string, Entry>, id: string): Partial<DiscoveredModel> {
|
||||
const exact = table[id];
|
||||
if (exact) return clone(exact);
|
||||
|
||||
// Longest known id that is a prefix of `id` and ends on a separator.
|
||||
let best = '';
|
||||
for (const key of Object.keys(table)) {
|
||||
if (key.length > best.length && id.startsWith(key) && /[-@:]/.test(id.charAt(key.length))) best = key;
|
||||
}
|
||||
if (!best) return {};
|
||||
const entry = table[best];
|
||||
if (!entry) return {};
|
||||
const out = clone(entry);
|
||||
delete out.label; // the label names the base model, not this variant
|
||||
if (!SNAPSHOT_SUFFIX.test(id.slice(best.length))) {
|
||||
// A genuinely different variant (`o3-deep-research`, `gpt-4o-audio-...`):
|
||||
// capabilities are a fair guess, the price is not.
|
||||
delete out.priceIn;
|
||||
delete out.priceOut;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Defaults for a model: exact id first, then the longest known id that prefixes
|
||||
* it (so dated snapshots match their base model). Returns `{}` when unknown.
|
||||
*/
|
||||
export function lookupModelDefaults(kind: ProviderKind, modelId: string): Partial<DiscoveredModel> {
|
||||
// Gemini's listing names models "models/<id>".
|
||||
const id = modelId.trim().replace(/^models\//, '');
|
||||
if (!id) return {};
|
||||
|
||||
if (kind === 'openai_compatible') {
|
||||
// Routers name models "<vendor>/<model>" (`anthropic/claude-sonnet-5-5`).
|
||||
// Borrow capabilities and limits from the vendor's table, never its prices.
|
||||
const bare = id.slice(id.lastIndexOf('/') + 1).replace(/:[a-z]+$/, '');
|
||||
for (const table of [OPENAI, ANTHROPIC, GEMINI]) {
|
||||
const hit = lookupIn(table, bare);
|
||||
if (Object.keys(hit).length > 0) {
|
||||
delete hit.priceIn;
|
||||
delete hit.priceOut;
|
||||
// Temperature limits are properties of the first-party APIs, not of the router.
|
||||
if (hit.capabilities) delete hit.capabilities.temperature;
|
||||
return hit;
|
||||
}
|
||||
}
|
||||
return {};
|
||||
}
|
||||
return lookupIn(TABLES[kind], id);
|
||||
}
|
||||
|
||||
/** Every catalogued model id for a provider (used as a fallback list when discovery is unavailable). */
|
||||
export function catalogModelIds(kind: ProviderKind): string[] {
|
||||
return Object.keys(TABLES[kind]);
|
||||
}
|
||||
138
server/ai/providers/common.ts
Normal file
138
server/ai/providers/common.ts
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
/**
|
||||
* Helpers shared by the adapters that are about our neutral contract rather
|
||||
* than about HTTP: tool-argument parsing, model-list post-processing, the
|
||||
* connection-test wrapper.
|
||||
*/
|
||||
import { jsonrepair } from 'jsonrepair';
|
||||
import { AIError, type DiscoveredModel, type ProviderKind, type TestResult, type Usage } from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { isRecord } from './http.ts';
|
||||
|
||||
/** Strict parse of tool-call arguments; `undefined` when they are not a JSON object. */
|
||||
export function parseArgsStrict(raw: string): Record<string, unknown> | undefined {
|
||||
const text = raw.trim();
|
||||
if (text === '') return {};
|
||||
try {
|
||||
const value: unknown = JSON.parse(text);
|
||||
return isRecord(value) ? value : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the JSON arguments of a completed tool call. Models occasionally emit
|
||||
* slightly broken JSON (trailing commas, unescaped newlines); `jsonrepair`
|
||||
* fixes those. Anything still unparseable is an `invalid_response`.
|
||||
*
|
||||
* Do NOT use this on a call that was cut off by the output-token limit —
|
||||
* repairing a truncated call would silently invent a shorter input.
|
||||
*/
|
||||
export function parseToolArguments(raw: unknown, provider: string, toolName: string): Record<string, unknown> {
|
||||
if (isRecord(raw)) return raw; // some servers send an object instead of a JSON string
|
||||
const text = typeof raw === 'string' ? raw : raw == null ? '' : String(raw);
|
||||
const strict = parseArgsStrict(text);
|
||||
if (strict) return strict;
|
||||
try {
|
||||
const repaired: unknown = JSON.parse(jsonrepair(text));
|
||||
if (isRecord(repaired)) return repaired;
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, {
|
||||
providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text,
|
||||
});
|
||||
}
|
||||
|
||||
/** Stable id for providers that do not issue tool-call ids themselves. */
|
||||
export function newToolCallId(prefix: string): string {
|
||||
return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`;
|
||||
}
|
||||
|
||||
export function dataUrl(mime: string, base64: string): string {
|
||||
return `data:${mime};base64,${base64}`;
|
||||
}
|
||||
|
||||
export function num(v: unknown): number {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
|
||||
}
|
||||
|
||||
export function str(v: unknown): string {
|
||||
return typeof v === 'string' ? v : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Usage convention used by every adapter:
|
||||
* inputTokens = ALL prompt tokens, including cached ones
|
||||
* cacheRead/cacheWriteTokens = the cached subsets of inputTokens
|
||||
* outputTokens = ALL generated tokens, including reasoning
|
||||
* reasoningTokens = the reasoning subset of outputTokens
|
||||
*/
|
||||
export function makeUsage(u: { input: number; output: number; cacheRead?: number; cacheWrite?: number; reasoning?: number }): Usage {
|
||||
const usage: Usage = { inputTokens: u.input, outputTokens: u.output };
|
||||
if (u.cacheRead) usage.cacheReadTokens = u.cacheRead;
|
||||
if (u.cacheWrite) usage.cacheWriteTokens = u.cacheWrite;
|
||||
if (u.reasoning) usage.reasoningTokens = u.reasoning;
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids that are obviously not text-chat models: embeddings, speech, image and
|
||||
* video generation, moderation, realtime/audio-only, rerankers, legacy
|
||||
* completion-only models.
|
||||
*/
|
||||
const NON_CHAT_RE = new RegExp(
|
||||
[
|
||||
'embed', // text-embedding-3-large, gemini-embedding-001, nomic-embed-text
|
||||
'\\btts\\b|whisper|transcri|speech|\\baudio\\b|realtime|\\blive\\b',
|
||||
'dall-e|gpt-image|image-generation|imagen|-image\\b|image-preview|\\bsora\\b|\\bveo\\b|lyria|stable-diffusion',
|
||||
'moderation|-guard\\b|rerank|\\baqa\\b',
|
||||
'computer-use|robotics|deep-research',
|
||||
'^(babbage|davinci|curie|ada)\\b|^text-|turbo-instruct',
|
||||
].join('|'),
|
||||
'i',
|
||||
);
|
||||
|
||||
export function isChatModelId(id: string): boolean {
|
||||
return !NON_CHAT_RE.test(id);
|
||||
}
|
||||
|
||||
/** Merges discovery results with catalog defaults; values the provider reported always win. */
|
||||
export function withCatalogDefaults(kind: ProviderKind, models: DiscoveredModel[]): DiscoveredModel[] {
|
||||
const seen = new Set<string>();
|
||||
const out: DiscoveredModel[] = [];
|
||||
for (const m of models) {
|
||||
if (!m.id || seen.has(m.id)) continue;
|
||||
seen.add(m.id);
|
||||
const d = lookupModelDefaults(kind, m.id);
|
||||
const merged: DiscoveredModel = { ...d, ...stripUndefined(m), id: m.id };
|
||||
if (d.capabilities || m.capabilities) merged.capabilities = { ...d.capabilities, ...stripUndefined(m.capabilities ?? {}) };
|
||||
out.push(merged);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function stripUndefined<T extends object>(obj: T): T {
|
||||
const out = {} as Record<string, unknown>;
|
||||
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null) out[k] = v;
|
||||
return out as T;
|
||||
}
|
||||
|
||||
/** Shared implementation of `ProviderAdapter.test()`: never throws, always friendly. */
|
||||
export async function runTest(label: string, check: () => Promise<{ modelCount?: number; note?: string }>): Promise<TestResult> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const { modelCount, note } = await check();
|
||||
const latencyMs = Date.now() - started;
|
||||
const message =
|
||||
note ?? (modelCount !== undefined ? `Connected to ${label} — ${modelCount} model${modelCount === 1 ? '' : 's'} available.` : `Connected to ${label}.`);
|
||||
return modelCount !== undefined ? { ok: true, message, latencyMs, modelCount } : { ok: true, message, latencyMs };
|
||||
} catch (err) {
|
||||
const latencyMs = Date.now() - started;
|
||||
const message = err instanceof AIError ? err.message : `Could not connect to ${label}.`;
|
||||
return { ok: false, message, latencyMs };
|
||||
}
|
||||
}
|
||||
|
||||
/** Timeout used by `test()` so a dead endpoint fails fast instead of hanging the settings page. */
|
||||
export const TEST_TIMEOUT_MS = 20_000;
|
||||
342
server/ai/providers/gemini.ts
Normal file
342
server/ai/providers/gemini.ts
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
/**
|
||||
* Google Gemini — `models/{model}:streamGenerateContent?alt=sse`.
|
||||
* https://ai.google.dev/api/generate-content
|
||||
* https://ai.google.dev/gemini-api/docs/function-calling
|
||||
* https://ai.google.dev/gemini-api/docs/thinking
|
||||
* https://ai.google.dev/gemini-api/docs/thought-signatures
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - The API key travels in the `x-goog-api-key` header, never in the URL.
|
||||
* - Function declarations accept only a subset of JSON Schema, so schemas are
|
||||
* down-converted (`toGeminiSchema`).
|
||||
* - Gemini issues no tool-call ids; stable ones are synthesised.
|
||||
* - Thinking models attach a `thoughtSignature` to parts (notably function calls)
|
||||
* that must be sent back unchanged, so each turn's raw parts live in `providerData`.
|
||||
* - Thinking is configured with `thinkingLevel` on Gemini 3 and later and with a
|
||||
* numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried.
|
||||
* - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY).
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { isChatModelId, makeUsage, newToolCallId, num, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'Google Gemini';
|
||||
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
|
||||
|
||||
function root(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1(beta)?$/, '');
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add a Gemini API key in Settings to use Gemini models.');
|
||||
return { ...cfg.headers, 'x-goog-api-key': cfg.apiKey };
|
||||
}
|
||||
|
||||
const SCHEMA_KEYS = new Set(['type', 'description', 'enum', 'properties', 'required', 'items', 'nullable', 'format', 'minimum', 'maximum', 'minItems', 'maxItems', 'anyOf', 'title']);
|
||||
const SCHEMA_FORMATS = new Set(['date-time', 'enum', 'int32', 'int64', 'float', 'double']);
|
||||
|
||||
/** Reduces JSON Schema to the OpenAPI-style subset Gemini's `parameters` field accepts. */
|
||||
export function toGeminiSchema(schema: unknown): Json {
|
||||
if (!isRecord(schema)) return { type: 'string' };
|
||||
const out: Json = {};
|
||||
for (const [key, value] of Object.entries(schema)) {
|
||||
if (!SCHEMA_KEYS.has(key)) continue; // additionalProperties, $schema, default, pattern, const, ...
|
||||
if (key === 'properties' && isRecord(value)) {
|
||||
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
|
||||
} else if (key === 'items') {
|
||||
out.items = toGeminiSchema(value);
|
||||
} else if (key === 'anyOf' && Array.isArray(value)) {
|
||||
out.anyOf = value.map(toGeminiSchema);
|
||||
} else if (key === 'type' && Array.isArray(value)) {
|
||||
// ["string","null"] -> nullable string
|
||||
const real = value.filter((t) => t !== 'null');
|
||||
out.type = real[0] ?? 'string';
|
||||
if (value.includes('null')) out.nullable = true;
|
||||
} else if (key === 'format') {
|
||||
if (typeof value === 'string' && SCHEMA_FORMATS.has(value)) out.format = value;
|
||||
} else {
|
||||
out[key] = value;
|
||||
}
|
||||
}
|
||||
if (out.enum && !out.type) out.type = 'string';
|
||||
if (out.type === 'object' && !out.properties) out.properties = {};
|
||||
return out;
|
||||
}
|
||||
|
||||
function userParts(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'tool_result') {
|
||||
out.push({ functionResponse: { name: p.name, response: p.isError ? { error: p.output } : { result: p.output } } });
|
||||
} else if (p.type === 'text' && p.text) {
|
||||
out.push({ text: p.text });
|
||||
} else if (p.type === 'image') {
|
||||
if (caps.vision === false) out.push({ text: '[An image was attached, but this model cannot read images.]' });
|
||||
else out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ inlineData: { mimeType: p.mime, data: p.data } });
|
||||
else if (p.mime.startsWith('text/')) out.push({ text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else out.push({ text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function toContents(messages: ChatMessage[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const contents: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
let parts: Json[];
|
||||
if (msg.role === 'assistant') {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'gemini' && isRecord(pd.data) && Array.isArray(pd.data.parts)) {
|
||||
parts = pd.data.parts.filter(isRecord); // thought signatures included, exactly as received
|
||||
} else {
|
||||
parts = [];
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'text' && p.text) parts.push({ text: p.text });
|
||||
else if (p.type === 'tool_call') parts.push({ functionCall: { name: p.name, args: p.input ?? {} } });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
parts = userParts(msg.content, caps);
|
||||
}
|
||||
if (parts.length === 0) continue;
|
||||
const role = msg.role === 'assistant' ? 'model' : 'user';
|
||||
const last = contents[contents.length - 1];
|
||||
// Gemini requires strictly alternating roles; merge consecutive turns of the same role.
|
||||
if (last && last.role === role) (last.parts as Json[]).push(...parts);
|
||||
else contents.push({ role, parts });
|
||||
}
|
||||
if (contents[0]?.role === 'model') contents.unshift({ role: 'user', parts: [{ text: '(continuing the conversation)' }] });
|
||||
return contents;
|
||||
}
|
||||
|
||||
type ThinkingStyle = 'level' | 'budget' | 'off';
|
||||
|
||||
function thinkingStyleFor(model: string): ThinkingStyle {
|
||||
const m = /gemini-(\d+)(?:\.(\d+))?/.exec(model);
|
||||
if (!m) return 'level';
|
||||
const major = Number(m[1]);
|
||||
if (major >= 3) return 'level';
|
||||
if (major === 2 && Number(m[2] ?? 0) >= 5) return 'budget';
|
||||
return 'off';
|
||||
}
|
||||
|
||||
const BUDGETS = { none: 0, low: 2_048, medium: 8_192, high: 24_576 } as const;
|
||||
|
||||
function buildBody(req: ChatRequest, thinking: ThinkingStyle, jsonMode: boolean): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { contents: toContents(req.messages, caps) };
|
||||
if (req.system?.trim()) body.systemInstruction = { parts: [{ text: req.system.trim() }] };
|
||||
|
||||
const hasTools = caps.tools !== false && (req.tools?.length ?? 0) > 0;
|
||||
if (hasTools) {
|
||||
body.tools = [{ functionDeclarations: req.tools!.map((t) => ({ name: t.name, description: t.description, parameters: toGeminiSchema({ type: 'object', ...t.parameters }) })) }];
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none') body.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
|
||||
else if (choice === 'required') body.toolConfig = { functionCallingConfig: { mode: 'ANY' } };
|
||||
else if (isRecord(choice)) body.toolConfig = { functionCallingConfig: { mode: 'ANY', allowedFunctionNames: [choice.name] } };
|
||||
}
|
||||
|
||||
const gen: Json = {};
|
||||
if (req.temperature !== undefined && caps.temperature !== false) gen.temperature = req.temperature;
|
||||
if (req.maxOutputTokens) gen.maxOutputTokens = req.maxOutputTokens;
|
||||
// JSON mode cannot be combined with function calling on most models.
|
||||
if (jsonMode && req.jsonMode && !hasTools) gen.responseMimeType = 'application/json';
|
||||
if (caps.reasoning !== false && thinking !== 'off') {
|
||||
const effort = req.reasoningEffort;
|
||||
if (thinking === 'level') {
|
||||
const cfg: Json = { includeThoughts: true };
|
||||
if (effort) cfg.thinkingLevel = effort === 'none' ? 'minimal' : effort;
|
||||
gen.thinkingConfig = cfg;
|
||||
} else if (effort) {
|
||||
// 2.5 Pro cannot switch thinking off; its minimum budget is 128 tokens.
|
||||
const budget = effort === 'none' && /pro/.test(req.model) ? 128 : BUDGETS[effort];
|
||||
gen.thinkingConfig = { thinkingBudget: budget, includeThoughts: budget > 0 };
|
||||
} else {
|
||||
gen.thinkingConfig = { includeThoughts: true };
|
||||
}
|
||||
}
|
||||
if (Object.keys(gen).length > 0) body.generationConfig = gen;
|
||||
return body;
|
||||
}
|
||||
|
||||
function mapFinish(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'STOP':
|
||||
return 'end';
|
||||
case 'MAX_TOKENS':
|
||||
return 'max_tokens';
|
||||
case 'SAFETY':
|
||||
case 'RECITATION':
|
||||
case 'BLOCKLIST':
|
||||
case 'PROHIBITED_CONTENT':
|
||||
case 'SPII':
|
||||
case 'IMAGE_SAFETY':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other'; // MALFORMED_FUNCTION_CALL, OTHER, UNEXPECTED_TOOL_CALL, ...
|
||||
}
|
||||
}
|
||||
|
||||
function modelPath(model: string): string {
|
||||
return model.startsWith('models/') || model.startsWith('tunedModels/') ? model : `models/${model}`;
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const url = `${root(cfg)}/v1beta/${modelPath(req.model)}:streamGenerateContent?alt=sse`;
|
||||
let thinking = thinkingStyleFor(req.model);
|
||||
let jsonMode = true;
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(url, { ...ctx, headers: headers(cfg), body: buildBody(req, thinking, jsonMode), signal: req.signal });
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (thinking !== 'off' && /thinking|thought/.test(detail)) thinking = thinking === 'level' ? 'budget' : 'off';
|
||||
else if (jsonMode && req.jsonMode && /response_?mime|json/.test(detail)) jsonMode = false;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const parts: Json[] = [];
|
||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
let text = '';
|
||||
let finish = '';
|
||||
let blocked = '';
|
||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
||||
if (isRecord(data.promptFeedback) && str(data.promptFeedback.blockReason)) blocked = str(data.promptFeedback.blockReason);
|
||||
if (isRecord(data.usageMetadata)) {
|
||||
const u = data.usageMetadata;
|
||||
usage.input = num(u.promptTokenCount) || usage.input;
|
||||
usage.reasoning = num(u.thoughtsTokenCount) || usage.reasoning;
|
||||
// candidatesTokenCount excludes thinking; both are billed as output.
|
||||
usage.output = num(u.candidatesTokenCount) + num(u.thoughtsTokenCount) || usage.output;
|
||||
usage.cacheRead = num(u.cachedContentTokenCount) || usage.cacheRead;
|
||||
}
|
||||
const candidate = Array.isArray(data.candidates) ? data.candidates[0] : undefined;
|
||||
if (!isRecord(candidate)) continue;
|
||||
if (str(candidate.finishReason)) finish = str(candidate.finishReason);
|
||||
const chunkParts = isRecord(candidate.content) && Array.isArray(candidate.content.parts) ? candidate.content.parts.filter(isRecord) : [];
|
||||
for (const part of chunkParts) {
|
||||
if (isRecord(part.functionCall)) {
|
||||
const call = { id: str(part.functionCall.id) || newToolCallId('gem'), name: str(part.functionCall.name), input: isRecord(part.functionCall.args) ? part.functionCall.args : {} };
|
||||
calls.push(call);
|
||||
parts.push(part);
|
||||
yield { type: 'tool_call', call };
|
||||
} else if (typeof part.text === 'string') {
|
||||
if (part.thought === true) {
|
||||
if (part.text) yield { type: 'reasoning', delta: part.text };
|
||||
// Thought summaries are display-only; only a signature needs to survive.
|
||||
if (part.thoughtSignature) parts.push({ text: '', thought: true, thoughtSignature: part.thoughtSignature });
|
||||
} else {
|
||||
if (part.text) {
|
||||
text += part.text;
|
||||
yield { type: 'text', delta: part.text };
|
||||
}
|
||||
// Merge streamed text fragments, but keep a part that carries a signature intact.
|
||||
const last = parts[parts.length - 1];
|
||||
if (last && typeof last.text === 'string' && last.thought !== true && !last.thoughtSignature && !part.thoughtSignature) last.text += part.text;
|
||||
else parts.push({ ...part });
|
||||
}
|
||||
} else {
|
||||
parts.push(part); // inlineData, executableCode, ... replayed untouched
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (blocked && !text && calls.length === 0) {
|
||||
throw new AIError('content_filter', 'Gemini blocked this request under its safety policy.', { providerMessage: `blockReason: ${blocked}` });
|
||||
}
|
||||
if (!finish && !text && calls.length === 0) {
|
||||
throw new AIError('network', 'The connection to Gemini closed before the response finished.', { retryable: true });
|
||||
}
|
||||
if (finish === 'MALFORMED_FUNCTION_CALL' && calls.length === 0) {
|
||||
throw new AIError('invalid_response', 'Gemini produced a malformed tool call. Try again.', { retryable: true });
|
||||
}
|
||||
|
||||
const content: ContentPart[] = [];
|
||||
if (text) content.push({ type: 'text', text });
|
||||
for (const call of calls) content.push({ type: 'tool_call', ...call });
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
const replay = parts.filter((p) => !(typeof p.text === 'string' && p.text === '' && !p.thoughtSignature));
|
||||
if (replay.length > 0) message.providerData = { kind: 'gemini', data: { parts: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: calls.length > 0 ? 'tool_use' : mapFinish(finish || 'STOP'),
|
||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
||||
};
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const found: DiscoveredModel[] = [];
|
||||
let pageToken = '';
|
||||
for (let page = 0; page < 10; page += 1) {
|
||||
const url = `${root(cfg)}/v1beta/models?pageSize=1000${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
|
||||
const res = await requestJson<Json>(url, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
for (const m of Array.isArray(res.models) ? res.models.filter(isRecord) : []) {
|
||||
const id = str(m.name).replace(/^models\//, '');
|
||||
const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods.map(String) : [];
|
||||
if (!id || !methods.includes('generateContent') || !isChatModelId(id)) continue;
|
||||
if (!/^(gemini|gemma|learnlm)/.test(id)) continue;
|
||||
const model: DiscoveredModel = { id };
|
||||
if (str(m.displayName)) model.label = str(m.displayName);
|
||||
if (num(m.inputTokenLimit)) model.contextWindow = num(m.inputTokenLimit);
|
||||
if (num(m.outputTokenLimit)) model.maxOutput = num(m.outputTokenLimit);
|
||||
if (typeof m.thinking === 'boolean') model.capabilities = { reasoning: m.thinking };
|
||||
if (id.startsWith('gemma')) model.capabilities = { ...model.capabilities, tools: false, pdf: false };
|
||||
found.push(model);
|
||||
}
|
||||
pageToken = str(res.nextPageToken);
|
||||
if (!pageToken) break;
|
||||
}
|
||||
return withCatalogDefaults('gemini', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const geminiAdapter: ProviderAdapter = {
|
||||
kind: 'gemini',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://aistudio.google.com/apikey',
|
||||
keyHint: 'AIza…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
519
server/ai/providers/http.ts
Normal file
519
server/ai/providers/http.ts
Normal file
|
|
@ -0,0 +1,519 @@
|
|||
/**
|
||||
* Shared HTTP plumbing for provider adapters: a fetch wrapper with timeouts,
|
||||
* an SSE parser over Web Streams, and one place that turns provider failures
|
||||
* into `AIError`s. Uses only global `fetch` — no SDKs.
|
||||
*
|
||||
* Security rule for this whole folder: an API key must never reach an error
|
||||
* message, a log line or a thrown value. Everything user-visible that contains
|
||||
* provider text goes through `scrubSecrets` first.
|
||||
*/
|
||||
import { AIError, type AIErrorCode } from '../types.ts';
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Secret scrubbing */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const KEY_PATTERNS: RegExp[] = [
|
||||
/\bsk-[A-Za-z0-9_-]{8,}/g, // OpenAI / Anthropic / OpenRouter style
|
||||
/\bAIza[0-9A-Za-z_-]{16,}/g, // Google API keys
|
||||
/\bgsk_[A-Za-z0-9]{8,}/g, // Groq
|
||||
/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
|
||||
/([?&](?:key|api_key|apikey|access_token)=)[^&\s"']+/gi,
|
||||
];
|
||||
|
||||
/** Removes known secrets (and anything that looks like an API key) from provider text. */
|
||||
export function scrubSecrets(text: string, secrets: Array<string | null | undefined> = []): string {
|
||||
let out = text;
|
||||
for (const s of secrets) {
|
||||
if (s && s.length >= 4) out = out.split(s).join('[redacted]');
|
||||
}
|
||||
for (const re of KEY_PATTERNS) {
|
||||
out = out.replace(re, (_m, prefix?: unknown) => (typeof prefix === 'string' ? `${prefix}[redacted]` : '[redacted]'));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function clip(text: string, max = 800): string {
|
||||
const t = text.trim();
|
||||
return t.length > max ? `${t.slice(0, max)}…` : t;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Error mapping */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface ErrorContext {
|
||||
/** Human provider label used in messages ("OpenAI", "Ollama"). */
|
||||
provider: string;
|
||||
/** Values that must never appear in an error (API key, custom header values). */
|
||||
secrets?: Array<string | null | undefined>;
|
||||
}
|
||||
|
||||
interface ProviderErrorFields {
|
||||
message: string;
|
||||
type: string;
|
||||
code: string;
|
||||
/** Whole body as text, used for keyword matching (quota ids, reasons, ...). */
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/** Pulls message/type/code out of the error envelopes used by the providers we talk to. */
|
||||
export function extractProviderError(body: unknown): ProviderErrorFields {
|
||||
let node: unknown = body;
|
||||
if (typeof node === 'string') {
|
||||
const text = node;
|
||||
try {
|
||||
node = JSON.parse(text);
|
||||
} catch {
|
||||
return { message: text.trim().startsWith('<') ? '' : text.trim(), type: '', code: '', raw: text };
|
||||
}
|
||||
}
|
||||
const raw = safeStringify(node);
|
||||
// Gemini sometimes wraps the error object in an array.
|
||||
if (Array.isArray(node)) node = node[0];
|
||||
if (!isRecord(node)) return { message: '', type: '', code: '', raw };
|
||||
const err = node.error ?? node;
|
||||
// Ollama: {"error": "model 'x' not found"}
|
||||
if (typeof err === 'string') return { message: err, type: '', code: '', raw };
|
||||
if (!isRecord(err)) return { message: '', type: '', code: '', raw };
|
||||
const message = typeof err.message === 'string' ? err.message : typeof node.message === 'string' ? node.message : '';
|
||||
// OpenAI: error.type + error.code; Anthropic: error.type; Gemini: error.status (+ numeric code).
|
||||
const type = [err.type, err.status].filter((v): v is string => typeof v === 'string').join(' ');
|
||||
const code = typeof err.code === 'string' ? err.code : '';
|
||||
return { message, type, code, raw };
|
||||
}
|
||||
|
||||
const CONTEXT_RE =
|
||||
/context[_ ]length|context[_ ]window|maximum context|max(?:imum)? (?:input )?tokens? (?:limit|allowed)|too many (?:input )?tokens|prompt is too long|input is too long|input token count|exceeds? the (?:model'?s? )?(?:maximum|max)[^.]{0,40}tokens|reduce the length|request[_ ]too[_ ]large|exceed_context_size|n_ctx/i;
|
||||
const QUOTA_RE =
|
||||
/insufficient[_ ]quota|billing|credit balance|exceeded your current quota|out of credits|insufficient (?:credits|funds|balance)|payment required|spend(?:ing)? limit/i;
|
||||
const AUTH_RE = /invalid[_ ]api[_ ]key|incorrect api key|api key not valid|api_key_invalid|authentication[_ ]error|invalid x-api-key|unauthorized|no auth credentials/i;
|
||||
const FILTER_RE = /content[_ ]filter|content[_ ]policy|content management policy|safety system|responsible ai/i;
|
||||
|
||||
function classify(status: number | undefined, f: ProviderErrorFields): AIErrorCode {
|
||||
const hay = `${f.type} ${f.code} ${f.message}`;
|
||||
const s = status ?? 0;
|
||||
|
||||
// Provider error names win over the status code: they are also all we have
|
||||
// for errors delivered inside an already-open (HTTP 200) stream.
|
||||
if (/authentication_error|invalid_api_key|permission_error|PERMISSION_DENIED|UNAUTHENTICATED/.test(hay)) return 'auth';
|
||||
if (/insufficient_quota|billing_error/.test(hay)) return 'quota';
|
||||
if (/context_length_exceeded|request_too_large/.test(hay)) return 'context_length';
|
||||
if (/overloaded_error|server_error|api_error|UNAVAILABLE|INTERNAL\b/.test(`${f.type} ${f.code}`)) return 'server';
|
||||
|
||||
if (s === 401 || s === 403) return 'auth';
|
||||
if (s === 402) return 'quota';
|
||||
if (s === 429 || /rate_limit|RESOURCE_EXHAUSTED/.test(hay)) {
|
||||
// Gemini words every 429 as "exceeded your current quota"; its details say
|
||||
// whether it is a per-minute limit (retry soon) or a daily/billing cap.
|
||||
if (/PerMinute|retryDelay|RetryInfo/.test(f.raw) && !/PerDay/.test(f.raw)) return 'rate_limit';
|
||||
return QUOTA_RE.test(hay) || /PerDay/.test(f.raw) ? 'quota' : 'rate_limit';
|
||||
}
|
||||
if (s === 404 || /not_found_error|model_not_found|NOT_FOUND/.test(hay)) return 'not_found';
|
||||
if (s === 408 || s === 504 || /DEADLINE_EXCEEDED/.test(hay)) return 'timeout';
|
||||
if (s === 413) return 'context_length';
|
||||
if (s >= 500) return 'server';
|
||||
// Remaining 4xx (and status-less stream errors).
|
||||
if (AUTH_RE.test(hay)) return 'auth'; // Gemini answers a bad key with 400 INVALID_ARGUMENT
|
||||
if (CONTEXT_RE.test(hay)) return 'context_length';
|
||||
if (QUOTA_RE.test(hay)) return 'quota'; // Anthropic: 400 "credit balance is too low"
|
||||
if (FILTER_RE.test(hay)) return 'content_filter';
|
||||
if (s === 0 && !f.message) return 'server';
|
||||
return 'bad_request';
|
||||
}
|
||||
|
||||
function friendlyMessage(code: AIErrorCode, provider: string): string {
|
||||
switch (code) {
|
||||
case 'auth':
|
||||
return `${provider} rejected the API key.`;
|
||||
case 'rate_limit':
|
||||
return `${provider} is rate limiting requests. Try again in a moment.`;
|
||||
case 'quota':
|
||||
return `${provider} reports that the account is out of quota or credit.`;
|
||||
case 'context_length':
|
||||
return `The conversation is too long for this ${provider} model.`;
|
||||
case 'not_found':
|
||||
return `${provider} could not find that model or endpoint.`;
|
||||
case 'content_filter':
|
||||
return `${provider} blocked the request with its content filter.`;
|
||||
case 'server':
|
||||
return `${provider} is having trouble right now. Try again shortly.`;
|
||||
case 'timeout':
|
||||
return `${provider} took too long to respond.`;
|
||||
case 'bad_request':
|
||||
return `${provider} rejected the request.`;
|
||||
default:
|
||||
return `${provider} returned an error.`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The single place that maps "HTTP status + provider error body" to an AIError.
|
||||
* Pass `status: undefined` for errors that arrive as events inside a stream.
|
||||
*/
|
||||
export function errorFromProvider(ctx: ErrorContext, status: number | undefined, body: unknown): AIError {
|
||||
const fields = extractProviderError(body);
|
||||
const code = classify(status, fields);
|
||||
const detail = fields.message || (status ? `HTTP ${status}` : '');
|
||||
return new AIError(code, friendlyMessage(code, ctx.provider), {
|
||||
status,
|
||||
providerMessage: detail ? clip(scrubSecrets(detail, ctx.secrets)) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
/** Maps anything thrown by `fetch` / a body read to an AIError. */
|
||||
export function errorFromThrown(ctx: ErrorContext, err: unknown, abortReason?: 'timeout' | 'aborted' | null): AIError {
|
||||
if (err instanceof AIError) return err;
|
||||
const name = isRecord(err) && typeof err.name === 'string' ? err.name : '';
|
||||
if (abortReason === 'timeout' || name === 'TimeoutError') {
|
||||
return new AIError('timeout', friendlyMessage('timeout', ctx.provider), { cause: err });
|
||||
}
|
||||
if (abortReason === 'aborted' || name === 'AbortError') {
|
||||
return new AIError('aborted', 'The request was cancelled.', { cause: err });
|
||||
}
|
||||
// undici reports DNS/connection failures as TypeError("fetch failed") with the real reason in `cause`.
|
||||
const cause = isRecord(err) && isRecord(err.cause) ? err.cause : undefined;
|
||||
const parts = [
|
||||
cause && typeof cause.code === 'string' ? cause.code : '',
|
||||
cause && typeof cause.message === 'string' ? cause.message : '',
|
||||
err instanceof Error ? err.message : String(err),
|
||||
].filter(Boolean);
|
||||
return new AIError('network', `Could not reach ${ctx.provider}. Check the address and your connection.`, {
|
||||
providerMessage: clip(scrubSecrets(parts.join(': '), ctx.secrets)),
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* fetch with timeouts */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface HttpOptions extends ErrorContext {
|
||||
method?: 'GET' | 'POST';
|
||||
headers?: Record<string, string>;
|
||||
/** Serialised as JSON when present. */
|
||||
body?: unknown;
|
||||
signal?: AbortSignal;
|
||||
/** Time allowed until response headers arrive (default 60s). */
|
||||
firstByteTimeoutMs?: number;
|
||||
/** Hard cap on the whole exchange (default 30 min — streams are long). */
|
||||
totalTimeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface HttpResponse {
|
||||
response: Response;
|
||||
ctx: ErrorContext;
|
||||
/** Abort the exchange from our side (used by the idle watchdog). */
|
||||
abort(reason: 'timeout' | 'aborted'): void;
|
||||
abortReason(): 'timeout' | 'aborted' | null;
|
||||
/** Clears timers/listeners. Safe to call more than once. */
|
||||
cleanup(): void;
|
||||
}
|
||||
|
||||
export const DEFAULT_FIRST_BYTE_TIMEOUT_MS = 60_000;
|
||||
export const DEFAULT_TOTAL_TIMEOUT_MS = 30 * 60_000;
|
||||
/** Longest silence tolerated inside an open stream before we give up. */
|
||||
export const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000;
|
||||
|
||||
/**
|
||||
* Sends a request and resolves once headers are in. Does NOT check `response.ok`
|
||||
* (callers decide; see `ensureOk`). Throws AIError for abort/timeout/network.
|
||||
*/
|
||||
export async function httpRequest(url: string, opts: HttpOptions): Promise<HttpResponse> {
|
||||
const ctx: ErrorContext = { provider: opts.provider, secrets: opts.secrets };
|
||||
if (opts.signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
|
||||
|
||||
const controller = new AbortController();
|
||||
let reason: 'timeout' | 'aborted' | null = null;
|
||||
const abort = (r: 'timeout' | 'aborted') => {
|
||||
if (reason) return;
|
||||
reason = r;
|
||||
controller.abort();
|
||||
};
|
||||
const onUserAbort = () => abort('aborted');
|
||||
opts.signal?.addEventListener('abort', onUserAbort, { once: true });
|
||||
|
||||
const firstByte = setTimeout(() => abort('timeout'), opts.firstByteTimeoutMs ?? DEFAULT_FIRST_BYTE_TIMEOUT_MS);
|
||||
const total = setTimeout(() => abort('timeout'), opts.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS);
|
||||
let cleaned = false;
|
||||
const cleanup = () => {
|
||||
if (cleaned) return;
|
||||
cleaned = true;
|
||||
clearTimeout(firstByte);
|
||||
clearTimeout(total);
|
||||
opts.signal?.removeEventListener('abort', onUserAbort);
|
||||
};
|
||||
|
||||
const headers: Record<string, string> = { ...opts.headers };
|
||||
let body: string | undefined;
|
||||
if (opts.body !== undefined) {
|
||||
body = JSON.stringify(opts.body);
|
||||
if (!hasHeader(headers, 'content-type')) headers['Content-Type'] = 'application/json';
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: opts.method ?? (body === undefined ? 'GET' : 'POST'),
|
||||
headers,
|
||||
body,
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(firstByte);
|
||||
return { response, ctx, abort, abortReason: () => reason, cleanup };
|
||||
} catch (err) {
|
||||
cleanup();
|
||||
throw errorFromThrown(ctx, err, reason);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads the body of a failed response and throws the mapped AIError. Returns normally when `ok`. */
|
||||
export async function ensureOk(http: HttpResponse): Promise<void> {
|
||||
if (http.response.ok) return;
|
||||
let text = '';
|
||||
try {
|
||||
text = await http.response.text();
|
||||
} catch {
|
||||
/* body unreadable — the status code alone still maps to something useful */
|
||||
} finally {
|
||||
http.cleanup();
|
||||
}
|
||||
throw errorFromProvider(http.ctx, http.response.status, text);
|
||||
}
|
||||
|
||||
/** Request + status check + JSON parse, for non-streaming calls such as model discovery. */
|
||||
export async function requestJson<T = unknown>(url: string, opts: HttpOptions): Promise<T> {
|
||||
const http = await httpRequest(url, opts);
|
||||
await ensureOk(http);
|
||||
try {
|
||||
const text = await http.response.text();
|
||||
try {
|
||||
return JSON.parse(text) as T;
|
||||
} catch {
|
||||
throw new AIError('invalid_response', `${opts.provider} sent a response that was not valid JSON.`, {
|
||||
providerMessage: clip(scrubSecrets(text, opts.secrets), 300),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
||||
} finally {
|
||||
http.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Server-sent events */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface SSEMessage {
|
||||
/** Value of the `event:` field, when the server sent one. */
|
||||
event?: string;
|
||||
/** All `data:` lines of the event joined with "\n". */
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface SSEOptions {
|
||||
/** Abort when no bytes arrive for this long. */
|
||||
idleTimeoutMs?: number;
|
||||
onIdle?: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a text/event-stream body. Handles `\n`, `\r\n` and bare `\r` line
|
||||
* endings (also when split across chunks), multi-line `data:`, comment lines,
|
||||
* a UTF-8 BOM, and a final event that is not followed by a blank line.
|
||||
* Iteration ends at end-of-stream or at the OpenAI-style `data: [DONE]` sentinel.
|
||||
*/
|
||||
export async function* parseSSE(stream: ReadableStream<Uint8Array>, opts: SSEOptions = {}): AsyncGenerator<SSEMessage> {
|
||||
const reader = stream.getReader();
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
let buffer = '';
|
||||
let dataLines: string[] = [];
|
||||
let eventName: string | undefined;
|
||||
let sawCR = false; // previous chunk ended in "\r": swallow a leading "\n"
|
||||
let first = true;
|
||||
|
||||
const takeLine = (line: string): SSEMessage | 'done' | null => {
|
||||
if (line === '') {
|
||||
// Blank line dispatches the event.
|
||||
if (dataLines.length === 0) {
|
||||
eventName = undefined;
|
||||
return null;
|
||||
}
|
||||
const msg: SSEMessage = { data: dataLines.join('\n') };
|
||||
if (eventName !== undefined) msg.event = eventName;
|
||||
dataLines = [];
|
||||
eventName = undefined;
|
||||
return msg.data === '[DONE]' ? 'done' : msg;
|
||||
}
|
||||
if (line.startsWith(':')) return null; // comment / keep-alive
|
||||
const colon = line.indexOf(':');
|
||||
const field = colon === -1 ? line : line.slice(0, colon);
|
||||
let value = colon === -1 ? '' : line.slice(colon + 1);
|
||||
if (value.startsWith(' ')) value = value.slice(1);
|
||||
if (field === 'data') dataLines.push(value);
|
||||
else if (field === 'event') eventName = value;
|
||||
// `id` and `retry` are irrelevant for one-shot completions.
|
||||
return null;
|
||||
};
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await readWithIdle(reader, opts);
|
||||
if (done) break;
|
||||
let chunk = decoder.decode(value, { stream: true });
|
||||
if (first && chunk.length > 0) {
|
||||
first = false;
|
||||
if (chunk.charCodeAt(0) === 0xfeff) chunk = chunk.slice(1);
|
||||
}
|
||||
if (sawCR && chunk.startsWith('\n')) chunk = chunk.slice(1);
|
||||
if (chunk.length > 0) sawCR = chunk.endsWith('\r');
|
||||
buffer += chunk;
|
||||
|
||||
let start = 0;
|
||||
for (;;) {
|
||||
const idx = indexOfLineEnd(buffer, start);
|
||||
if (idx === -1) break;
|
||||
const line = buffer.slice(start, idx);
|
||||
start = buffer[idx] === '\r' && buffer[idx + 1] === '\n' ? idx + 2 : idx + 1;
|
||||
const out = takeLine(line);
|
||||
if (out === 'done') return;
|
||||
if (out) yield out;
|
||||
}
|
||||
buffer = buffer.slice(start);
|
||||
}
|
||||
// Flush: tolerate a last line / last event without the terminating newline(s).
|
||||
buffer += decoder.decode();
|
||||
if (buffer.length > 0) {
|
||||
const out = takeLine(buffer);
|
||||
if (out === 'done') return;
|
||||
if (out) yield out;
|
||||
}
|
||||
const tail = takeLine('');
|
||||
if (tail && tail !== 'done') yield tail;
|
||||
} finally {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already closed or errored */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function indexOfLineEnd(s: string, from: number): number {
|
||||
for (let i = from; i < s.length; i++) {
|
||||
const c = s.charCodeAt(i);
|
||||
if (c === 10 || c === 13) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
async function readWithIdle(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
opts: SSEOptions,
|
||||
): Promise<Awaited<ReturnType<ReadableStreamDefaultReader<Uint8Array>['read']>>> {
|
||||
if (!opts.idleTimeoutMs) return reader.read();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const idle = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
opts.onIdle?.();
|
||||
const err = new Error('stream idle timeout');
|
||||
err.name = 'TimeoutError';
|
||||
reject(err);
|
||||
}, opts.idleTimeoutMs);
|
||||
});
|
||||
try {
|
||||
return await Promise.race([reader.read(), idle]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterates the SSE events of an open response, translating read failures
|
||||
* (dropped connection, abort, idle timeout) into AIErrors and always releasing
|
||||
* the request's timers. `data` is JSON-parsed; unparseable events are skipped.
|
||||
*/
|
||||
export async function* sseJson(http: HttpResponse, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS): AsyncGenerator<{ event?: string; data: unknown }> {
|
||||
const body = http.response.body;
|
||||
if (!body) {
|
||||
http.cleanup();
|
||||
throw new AIError('invalid_response', `${http.ctx.provider} returned an empty response.`);
|
||||
}
|
||||
const events = parseSSE(body, { idleTimeoutMs, onIdle: () => http.abort('timeout') });
|
||||
try {
|
||||
for (;;) {
|
||||
let next: IteratorResult<SSEMessage>;
|
||||
try {
|
||||
next = await events.next();
|
||||
} catch (err) {
|
||||
throw errorFromThrown(http.ctx, err, http.abortReason());
|
||||
}
|
||||
if (next.done) return;
|
||||
let data: unknown;
|
||||
try {
|
||||
data = JSON.parse(next.value.data);
|
||||
} catch {
|
||||
continue; // keep-alives such as OpenRouter's ": OPENROUTER PROCESSING" variants, or junk
|
||||
}
|
||||
yield next.value.event !== undefined ? { event: next.value.event, data } : { data };
|
||||
}
|
||||
} finally {
|
||||
await events.return(undefined).catch(() => undefined);
|
||||
http.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Small shared utilities */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export function isRecord(v: unknown): v is Record<string, unknown> {
|
||||
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
||||
}
|
||||
|
||||
function hasHeader(headers: Record<string, string>, name: string): boolean {
|
||||
const n = name.toLowerCase();
|
||||
return Object.keys(headers).some((k) => k.toLowerCase() === n);
|
||||
}
|
||||
|
||||
function safeStringify(v: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(v) ?? '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/** Trims whitespace and trailing slashes from a base URL. */
|
||||
export function trimBaseUrl(url: string): string {
|
||||
return url.trim().replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/** Host part of a URL for user-facing messages; never includes credentials or query. */
|
||||
export function hostOf(url: string): string {
|
||||
try {
|
||||
return new URL(url).host;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
/** True for loopback / private-network hosts, where a local model server may need minutes to load a model. */
|
||||
export function isLocalUrl(url: string): boolean {
|
||||
try {
|
||||
const h = new URL(url).hostname.replace(/^\[|\]$/g, '');
|
||||
return (
|
||||
h === 'localhost' ||
|
||||
h === '::1' ||
|
||||
h.endsWith('.local') ||
|
||||
h.endsWith('.internal') ||
|
||||
/^127\./.test(h) ||
|
||||
/^10\./.test(h) ||
|
||||
/^192\.168\./.test(h) ||
|
||||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
|
||||
h === 'host.docker.internal'
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
26
server/ai/providers/index.ts
Normal file
26
server/ai/providers/index.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { AIError, type ProviderAdapter, type ProviderKind } from '../types.ts';
|
||||
import { anthropicAdapter } from './anthropic.ts';
|
||||
import { geminiAdapter } from './gemini.ts';
|
||||
import { ollamaAdapter } from './ollama.ts';
|
||||
import { openaiAdapter } from './openai.ts';
|
||||
import { openaiCompatibleAdapter } from './openaiCompatible.ts';
|
||||
|
||||
const ADAPTERS: Record<ProviderKind, ProviderAdapter> = {
|
||||
openai: openaiAdapter,
|
||||
anthropic: anthropicAdapter,
|
||||
gemini: geminiAdapter,
|
||||
openai_compatible: openaiCompatibleAdapter,
|
||||
ollama: ollamaAdapter,
|
||||
};
|
||||
|
||||
export function getAdapter(kind: ProviderKind): ProviderAdapter {
|
||||
const adapter = ADAPTERS[kind];
|
||||
if (!adapter) throw new AIError('unsupported', `Unknown provider type "${kind}".`);
|
||||
return adapter;
|
||||
}
|
||||
|
||||
export function listAdapterMeta(): Array<{ kind: ProviderKind } & ProviderAdapter['meta']> {
|
||||
return (Object.keys(ADAPTERS) as ProviderKind[]).map((kind) => ({ kind, ...ADAPTERS[kind].meta }));
|
||||
}
|
||||
|
||||
export { CATALOG_UPDATED, lookupModelDefaults } from './catalog.ts';
|
||||
102
server/ai/providers/ollama.ts
Normal file
102
server/ai/providers/ollama.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
/**
|
||||
* Ollama (local models). Chat goes through Ollama's OpenAI-compatible endpoint
|
||||
* (`{base}/v1/chat/completions`), which handles tools and streaming; discovery
|
||||
* uses the native API because it reports what each model can do:
|
||||
* GET /api/tags -> installed models
|
||||
* POST /api/show -> { capabilities: ["completion","tools","vision","thinking"], model_info }
|
||||
* https://docs.ollama.com/api · https://docs.ollama.com/openai
|
||||
*/
|
||||
import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts';
|
||||
import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts';
|
||||
import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts';
|
||||
import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts';
|
||||
|
||||
const LABEL = 'Ollama';
|
||||
const DEFAULT_BASE_URL = 'http://localhost:11434';
|
||||
|
||||
function root(cfg: ProviderConfig): string {
|
||||
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/(v1|api)$/, '');
|
||||
}
|
||||
|
||||
function ctx(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
const h: Record<string, string> = { ...cfg.headers };
|
||||
if (cfg.apiKey) h.Authorization = `Bearer ${cfg.apiKey}`; // only needed behind an authenticating proxy
|
||||
return h;
|
||||
}
|
||||
|
||||
function compat(cfg: ProviderConfig): CompatOptions {
|
||||
return { kind: 'ollama', label: LABEL, baseUrl: `${root(cfg)}/v1`, apiKey: cfg.apiKey, headers: cfg.headers };
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
let tags: unknown;
|
||||
try {
|
||||
tags = await requestJson(`${root(cfg)}/api/tags`, { ...ctx(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs ?? 15_000, totalTimeoutMs: timeoutMs ?? 30_000 });
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'network') {
|
||||
throw new AIError('network', `Could not reach Ollama at ${root(cfg)}. Is it running, and reachable from the Study Desk container?`, { retryable: true });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const list = isRecord(tags) && Array.isArray(tags.models) ? tags.models.filter(isRecord) : [];
|
||||
const models: DiscoveredModel[] = [];
|
||||
// /api/show is one cheap local call per model; cap it so a huge library cannot stall the settings page.
|
||||
await Promise.all(
|
||||
list.slice(0, 60).map(async (m) => {
|
||||
const id = str(m.model) || str(m.name);
|
||||
if (!id) return;
|
||||
const model: DiscoveredModel = { id, label: str(m.name) || id, priceIn: 0, priceOut: 0 };
|
||||
try {
|
||||
const show = await requestJson<Record<string, unknown>>(`${root(cfg)}/api/show`, {
|
||||
...ctx(cfg),
|
||||
headers: headers(cfg),
|
||||
body: { model: id },
|
||||
signal,
|
||||
firstByteTimeoutMs: 10_000,
|
||||
totalTimeoutMs: 10_000,
|
||||
});
|
||||
if (Array.isArray(show.capabilities)) {
|
||||
const caps = show.capabilities.map(String);
|
||||
if (!caps.includes('completion')) return; // embedding-only model
|
||||
const c: Partial<ModelCapabilities> = { tools: caps.includes('tools'), vision: caps.includes('vision'), reasoning: caps.includes('thinking'), pdf: false, temperature: true };
|
||||
model.capabilities = c;
|
||||
}
|
||||
if (isRecord(show.model_info)) {
|
||||
const key = Object.keys(show.model_info).find((k) => k.endsWith('.context_length'));
|
||||
if (key && num(show.model_info[key])) model.contextWindow = num(show.model_info[key]);
|
||||
}
|
||||
} catch {
|
||||
/* older Ollama versions: keep the model with unknown capabilities */
|
||||
}
|
||||
models.push(model);
|
||||
}),
|
||||
);
|
||||
return models.sort((a, b) => a.id.localeCompare(b.id));
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => {
|
||||
const count = (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length;
|
||||
return count === 0 ? { modelCount: 0, note: 'Connected to Ollama, but no models are installed. Run `ollama pull <model>` first.' } : { modelCount: count };
|
||||
});
|
||||
}
|
||||
|
||||
export const ollamaAdapter: ProviderAdapter = {
|
||||
kind: 'ollama',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: false,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://docs.ollama.com',
|
||||
keyHint: 'Not needed',
|
||||
},
|
||||
stream: (cfg, req) => streamChatCompletions(compat(cfg), req),
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
313
server/ai/providers/openai.ts
Normal file
313
server/ai/providers/openai.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* OpenAI — Responses API (`POST /v1/responses`, SSE).
|
||||
* https://platform.openai.com/docs/api-reference/responses
|
||||
* https://platform.openai.com/docs/guides/function-calling
|
||||
* https://platform.openai.com/docs/guides/reasoning
|
||||
*
|
||||
* Quirks handled here:
|
||||
* - Requests are stateless (`store: false`). Reasoning models need their reasoning
|
||||
* items passed back between tool calls, so encrypted reasoning is requested via
|
||||
* `include: ["reasoning.encrypted_content"]` and every turn's raw output items are
|
||||
* kept in `providerData` and replayed verbatim.
|
||||
* - Reasoning models reject `temperature`; classic models reject `reasoning`.
|
||||
* - The accepted `reasoning.effort` values differ by model generation. When a model
|
||||
* rejects the chosen value the request is retried once with a safe one.
|
||||
* - Tool-call arguments arrive as deltas keyed by output item.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
const LABEL = 'OpenAI';
|
||||
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
|
||||
|
||||
function base(cfg: ProviderConfig): string {
|
||||
const url = trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL);
|
||||
return /\/v\d+$/.test(url) ? url : `${url}/v1`;
|
||||
}
|
||||
|
||||
function errorContext(cfg: ProviderConfig): ErrorContext {
|
||||
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
|
||||
}
|
||||
|
||||
function headers(cfg: ProviderConfig): Record<string, string> {
|
||||
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an OpenAI API key in Settings to use OpenAI models.');
|
||||
return { ...cfg.headers, Authorization: `Bearer ${cfg.apiKey}` };
|
||||
}
|
||||
|
||||
/** Whether a model is a reasoning model, when the caller did not say. */
|
||||
function isReasoningModel(req: ChatRequest): boolean {
|
||||
if (req.capabilities?.reasoning !== undefined) return req.capabilities.reasoning;
|
||||
const known = lookupModelDefaults('openai', req.model).capabilities?.reasoning;
|
||||
if (known !== undefined) return known;
|
||||
return /^(o\d|gpt-[5-9]|gpt-\d{2})/.test(req.model) && !/chat/.test(req.model);
|
||||
}
|
||||
|
||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
|
||||
const out: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'text' && p.text) out.push({ type: 'input_text', text: p.text });
|
||||
else if (p.type === 'image') {
|
||||
if (caps.vision === false) out.push({ type: 'input_text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else out.push({ type: 'input_image', image_url: dataUrl(p.mime, p.data) });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ type: 'input_file', filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) });
|
||||
else if (p.mime.startsWith('text/')) out.push({ type: 'input_text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else out.push({ type: 'input_text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function toInput(messages: ChatMessage[], caps: Partial<ModelCapabilities>, dropReasoning: boolean): Json[] {
|
||||
const input: Json[] = [];
|
||||
for (const msg of messages) {
|
||||
if (msg.role === 'assistant') {
|
||||
const pd = msg.providerData;
|
||||
if (pd?.kind === 'openai' && isRecord(pd.data) && Array.isArray(pd.data.output)) {
|
||||
// Same provider: replay the output items exactly (reasoning items included).
|
||||
for (const item of pd.data.output.filter(isRecord)) {
|
||||
if (item.type === 'reasoning' && (dropReasoning || !item.encrypted_content)) continue;
|
||||
input.push(item);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
||||
if (text) input.push({ role: 'assistant', content: [{ type: 'output_text', text }] });
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_call') input.push({ type: 'function_call', call_id: p.id, name: p.name, arguments: JSON.stringify(p.input ?? {}) });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_result') input.push({ type: 'function_call_output', call_id: p.toolCallId, output: p.output || '(no output)' });
|
||||
}
|
||||
const content = userContent(msg.content.filter((p) => p.type !== 'tool_result'), caps);
|
||||
if (content.length > 0) input.push({ role: 'user', content });
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
interface Flags {
|
||||
/** Value sent as reasoning.effort, or null to omit the field. */
|
||||
effort: string | null;
|
||||
summary: boolean;
|
||||
temperature: boolean;
|
||||
dropReasoning: boolean;
|
||||
include: boolean;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, flags: Flags): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { model: req.model, input: toInput(req.messages, caps, flags.dropReasoning), stream: true, store: false };
|
||||
if (req.system?.trim()) body.instructions = req.system.trim();
|
||||
if (req.maxOutputTokens) body.max_output_tokens = Math.max(16, req.maxOutputTokens);
|
||||
if (caps.tools !== false && req.tools?.length) {
|
||||
body.tools = req.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters }, strict: false }));
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', name: choice.name };
|
||||
}
|
||||
if (isReasoningModel(req)) {
|
||||
const reasoning: Json = {};
|
||||
if (flags.effort) reasoning.effort = flags.effort;
|
||||
if (flags.summary) reasoning.summary = 'auto';
|
||||
if (Object.keys(reasoning).length > 0) body.reasoning = reasoning;
|
||||
if (flags.include) body.include = ['reasoning.encrypted_content'];
|
||||
} else if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) {
|
||||
body.temperature = req.temperature;
|
||||
}
|
||||
if (req.jsonMode) body.text = { format: { type: 'json_object' } };
|
||||
return body;
|
||||
}
|
||||
|
||||
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(cfg);
|
||||
const flags: Flags = {
|
||||
// "none" is only understood by recent models; if it is rejected we fall back to "low".
|
||||
effort: req.reasoningEffort ?? null,
|
||||
summary: true,
|
||||
temperature: true,
|
||||
dropReasoning: false,
|
||||
include: true,
|
||||
};
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(`${base(cfg)}/responses`, { ...ctx, headers: headers(cfg), body: buildBody(req, flags), signal: req.signal });
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 5) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (flags.effort && /reasoning\.effort|reasoning_effort|effort/.test(detail)) flags.effort = flags.effort === 'none' ? 'minimal' : flags.effort === 'minimal' ? 'low' : null;
|
||||
else if (flags.summary && /summary/.test(detail)) flags.summary = false;
|
||||
else if (flags.include && /include|encrypted_content/.test(detail)) flags.include = false;
|
||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
||||
else if (!flags.dropReasoning && /reasoning/.test(detail) && /item|input/.test(detail)) flags.dropReasoning = true;
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const args = new Map<string, string>();
|
||||
let final: Json | null = null;
|
||||
let reasoningEmitted = false;
|
||||
let streamedText = '';
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
switch (data.type) {
|
||||
case 'response.output_text.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) {
|
||||
streamedText += delta;
|
||||
yield { type: 'text', delta };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'response.refusal.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) yield { type: 'text', delta };
|
||||
break;
|
||||
}
|
||||
case 'response.reasoning_summary_part.added':
|
||||
if (reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
|
||||
break;
|
||||
case 'response.reasoning_summary_text.delta':
|
||||
case 'response.reasoning_text.delta': {
|
||||
const delta = str(data.delta);
|
||||
if (delta) {
|
||||
reasoningEmitted = true;
|
||||
yield { type: 'reasoning', delta };
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'response.function_call_arguments.delta':
|
||||
args.set(str(data.item_id), (args.get(str(data.item_id)) ?? '') + str(data.delta));
|
||||
break;
|
||||
case 'response.completed':
|
||||
case 'response.incomplete':
|
||||
if (isRecord(data.response)) final = data.response;
|
||||
break;
|
||||
case 'response.failed':
|
||||
throw errorFromProvider(ctx, undefined, isRecord(data.response) && isRecord(data.response.error) ? { error: data.response.error } : data);
|
||||
case 'error':
|
||||
throw errorFromProvider(ctx, undefined, isRecord(data.error) ? data : { error: data });
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!final) throw new AIError('network', 'The connection to OpenAI closed before the response finished.', { retryable: true });
|
||||
|
||||
const incomplete = isRecord(final.incomplete_details) ? str(final.incomplete_details.reason) : '';
|
||||
const truncated = incomplete === 'max_output_tokens';
|
||||
const output = Array.isArray(final.output) ? final.output.filter(isRecord) : [];
|
||||
const content: ContentPart[] = [];
|
||||
const replay: Json[] = [];
|
||||
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
let refused = false;
|
||||
let text = '';
|
||||
|
||||
for (const item of output) {
|
||||
if (item.type === 'message') {
|
||||
for (const part of Array.isArray(item.content) ? item.content.filter(isRecord) : []) {
|
||||
if (part.type === 'output_text') text += str(part.text);
|
||||
else if (part.type === 'refusal') {
|
||||
refused = true;
|
||||
text += str(part.refusal);
|
||||
}
|
||||
}
|
||||
replay.push(item);
|
||||
} else if (item.type === 'function_call') {
|
||||
const name = str(item.name);
|
||||
const raw = str(item.arguments) || args.get(str(item.id)) || '';
|
||||
let input = parseArgsStrict(raw);
|
||||
if (!input) {
|
||||
if (truncated) continue;
|
||||
input = parseToolArguments(raw, LABEL, name);
|
||||
}
|
||||
calls.push({ id: str(item.call_id) || str(item.id), name, input });
|
||||
replay.push(item);
|
||||
} else if (item.type === 'reasoning') {
|
||||
replay.push(item);
|
||||
}
|
||||
}
|
||||
if (!text && streamedText) text = streamedText;
|
||||
if (text) content.push({ type: 'text', text });
|
||||
for (const call of calls) {
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
yield { type: 'tool_call', call };
|
||||
}
|
||||
|
||||
let stopReason: StopReason = 'end';
|
||||
if (calls.length > 0) stopReason = 'tool_use';
|
||||
else if (refused || incomplete === 'content_filter') stopReason = 'refusal';
|
||||
else if (truncated) stopReason = 'max_tokens';
|
||||
else if (incomplete) stopReason = 'other';
|
||||
|
||||
const u = isRecord(final.usage) ? final.usage : {};
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
if (replay.length > 0) message.providerData = { kind: 'openai', data: { output: replay } };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason,
|
||||
usage: makeUsage({
|
||||
input: num(u.input_tokens),
|
||||
output: num(u.output_tokens),
|
||||
cacheRead: isRecord(u.input_tokens_details) ? num(u.input_tokens_details.cached_tokens) : 0,
|
||||
reasoning: isRecord(u.output_tokens_details) ? num(u.output_tokens_details.reasoning_tokens) : 0,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const res = await requestJson<Json>(`${base(cfg)}/models`, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
|
||||
const found: DiscoveredModel[] = [];
|
||||
for (const m of data) {
|
||||
const id = str(m.id);
|
||||
// Fine-tunes ("ft:...") are chat models too; search/codex-only variants are not useful here.
|
||||
if (!id || !isChatModelId(id) || /search|^codex-|-codex-max/.test(id)) continue;
|
||||
if (!/^(gpt|o\d|chat|ft:)/.test(id)) continue;
|
||||
found.push({ id });
|
||||
}
|
||||
found.sort((a, b) => b.id.localeCompare(a.id, undefined, { numeric: true }));
|
||||
return withCatalogDefaults('openai', found);
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig): Promise<TestResult> {
|
||||
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
|
||||
}
|
||||
|
||||
export const openaiAdapter: ProviderAdapter = {
|
||||
kind: 'openai',
|
||||
meta: {
|
||||
label: LABEL,
|
||||
defaultBaseUrl: DEFAULT_BASE_URL,
|
||||
requiresApiKey: true,
|
||||
requiresBaseUrl: false,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://platform.openai.com/api-keys',
|
||||
keyHint: 'sk-…',
|
||||
},
|
||||
stream,
|
||||
listModels: (cfg, signal) => listModels(cfg, signal),
|
||||
test,
|
||||
};
|
||||
363
server/ai/providers/openaiCompatible.ts
Normal file
363
server/ai/providers/openaiCompatible.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
/**
|
||||
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
|
||||
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
|
||||
* Ollama adapter.
|
||||
*
|
||||
* These servers agree on the basics and disagree on everything else, so this
|
||||
* adapter is deliberately forgiving:
|
||||
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
|
||||
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
|
||||
* once when a server answers 400 about them;
|
||||
* - usage may be missing, and may arrive on a chunk with no choices;
|
||||
* - tool-call argument fragments may arrive in any chunking, with or without ids;
|
||||
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
|
||||
* - the last chunk may lack a `finish_reason`.
|
||||
*/
|
||||
import {
|
||||
AIError,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type ContentPart,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderAdapter,
|
||||
type ProviderConfig,
|
||||
type ProviderKind,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type TestResult,
|
||||
} from '../types.ts';
|
||||
import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
|
||||
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
|
||||
|
||||
type Json = Record<string, unknown>;
|
||||
|
||||
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
|
||||
export function normalizeCompatBase(url: string): string {
|
||||
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
|
||||
try {
|
||||
const u = new URL(base);
|
||||
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
|
||||
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
|
||||
} catch {
|
||||
/* validated by the caller */
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
export interface CompatOptions {
|
||||
kind: ProviderKind;
|
||||
label: string;
|
||||
baseUrl: string;
|
||||
apiKey: string | null;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
|
||||
const blocks: Json[] = [];
|
||||
for (const p of parts) {
|
||||
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
|
||||
else if (p.type === 'image') {
|
||||
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
|
||||
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
|
||||
} else if (p.type === 'file') {
|
||||
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
|
||||
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
|
||||
}
|
||||
}
|
||||
// Plain strings are understood by every server; arrays only by multimodal ones.
|
||||
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
|
||||
return blocks;
|
||||
}
|
||||
|
||||
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
|
||||
const caps = req.capabilities ?? {};
|
||||
const out: Json[] = [];
|
||||
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
|
||||
for (const msg of req.messages) {
|
||||
if (msg.role === 'assistant') {
|
||||
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
||||
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
|
||||
if (!text && calls.length === 0) continue;
|
||||
const wire: Json = { role: 'assistant', content: text || null };
|
||||
if (calls.length > 0) {
|
||||
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
|
||||
}
|
||||
out.push(wire);
|
||||
continue;
|
||||
}
|
||||
// Tool results must directly follow the assistant turn that requested them.
|
||||
for (const p of msg.content) {
|
||||
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
|
||||
}
|
||||
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
|
||||
if (rest.length > 0) {
|
||||
const content = userContent(rest, caps);
|
||||
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface BodyFlags {
|
||||
tools: boolean;
|
||||
usage: boolean;
|
||||
jsonMode: boolean;
|
||||
temperature: boolean;
|
||||
reasoning: boolean;
|
||||
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
|
||||
}
|
||||
|
||||
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
|
||||
const caps = req.capabilities ?? {};
|
||||
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
|
||||
if (flags.usage) body.stream_options = { include_usage: true };
|
||||
if (flags.tools && caps.tools !== false && req.tools?.length) {
|
||||
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
|
||||
const choice = req.toolChoice;
|
||||
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
|
||||
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
|
||||
}
|
||||
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
|
||||
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
|
||||
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
|
||||
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
|
||||
return body;
|
||||
}
|
||||
|
||||
function mapFinish(reason: string): StopReason {
|
||||
switch (reason) {
|
||||
case 'stop':
|
||||
case 'eos':
|
||||
case 'end_turn':
|
||||
return 'end';
|
||||
case 'tool_calls':
|
||||
case 'function_call':
|
||||
return 'tool_use';
|
||||
case 'length':
|
||||
case 'max_tokens':
|
||||
return 'max_tokens';
|
||||
case 'content_filter':
|
||||
return 'refusal';
|
||||
default:
|
||||
return 'other';
|
||||
}
|
||||
}
|
||||
|
||||
function authHeaders(o: CompatOptions): Record<string, string> {
|
||||
const h: Record<string, string> = { ...o.headers };
|
||||
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
function errorContext(o: CompatOptions): ErrorContext {
|
||||
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
|
||||
}
|
||||
|
||||
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
|
||||
const ctx = errorContext(o);
|
||||
const base = normalizeCompatBase(o.baseUrl);
|
||||
const local = isLocalUrl(base);
|
||||
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
|
||||
|
||||
let http: HttpResponse;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
http = await httpRequest(`${base}/chat/completions`, {
|
||||
...ctx,
|
||||
headers: authHeaders(o),
|
||||
body: buildBody(req, flags),
|
||||
signal: req.signal,
|
||||
// A local server may spend minutes loading a model into memory before the first byte.
|
||||
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
|
||||
});
|
||||
try {
|
||||
await ensureOk(http);
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
|
||||
const detail = (err.providerMessage ?? '').toLowerCase();
|
||||
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
|
||||
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
|
||||
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
|
||||
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
|
||||
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
|
||||
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
|
||||
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
|
||||
// The model (or server) cannot do tool calling at all.
|
||||
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
|
||||
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
|
||||
status: err.status,
|
||||
providerMessage: err.providerMessage,
|
||||
});
|
||||
}
|
||||
flags.tools = false;
|
||||
} else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
let text = '';
|
||||
let finish = '';
|
||||
let reasoningSeen = false;
|
||||
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
|
||||
const calls: { id: string; name: string; args: string }[] = [];
|
||||
|
||||
for await (const { data } of sseJson(http)) {
|
||||
if (!isRecord(data)) continue;
|
||||
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
|
||||
if (isRecord(data.usage)) {
|
||||
const u = data.usage;
|
||||
usage.input = num(u.prompt_tokens) || usage.input;
|
||||
usage.output = num(u.completion_tokens) || usage.output;
|
||||
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
|
||||
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
|
||||
}
|
||||
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
|
||||
if (!isRecord(choice)) continue;
|
||||
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
|
||||
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
|
||||
if (!delta) continue;
|
||||
|
||||
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
|
||||
if (reasoning) {
|
||||
reasoningSeen = true;
|
||||
yield { type: 'reasoning', delta: reasoning };
|
||||
}
|
||||
const piece = typeof delta.content === 'string' ? delta.content : '';
|
||||
if (piece) {
|
||||
text += piece;
|
||||
yield { type: 'text', delta: piece };
|
||||
}
|
||||
if (Array.isArray(delta.tool_calls)) {
|
||||
for (const raw of delta.tool_calls) {
|
||||
if (!isRecord(raw)) continue;
|
||||
const fn = isRecord(raw.function) ? raw.function : {};
|
||||
// `index` identifies the call; servers that omit it send one whole call per entry.
|
||||
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
|
||||
let slot = index >= 0 ? calls[index] : undefined;
|
||||
if (!slot) {
|
||||
slot = { id: '', name: '', args: '' };
|
||||
if (index >= 0) calls[index] = slot;
|
||||
else calls.push(slot);
|
||||
}
|
||||
if (str(raw.id)) slot.id = str(raw.id);
|
||||
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
|
||||
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
|
||||
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const stop = finish ? mapFinish(finish) : 'end';
|
||||
const content: ContentPart[] = [];
|
||||
if (text) content.push({ type: 'text', text });
|
||||
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
|
||||
for (const c of calls) {
|
||||
if (!c || !c.name) continue;
|
||||
let input = parseArgsStrict(c.args);
|
||||
if (!input) {
|
||||
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
|
||||
input = parseToolArguments(c.args, o.label, c.name);
|
||||
}
|
||||
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
|
||||
}
|
||||
for (const call of done) {
|
||||
content.push({ type: 'tool_call', ...call });
|
||||
yield { type: 'tool_call', call };
|
||||
}
|
||||
if (content.length === 0 && !reasoningSeen && !finish) {
|
||||
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
|
||||
}
|
||||
const message: ChatMessage = { role: 'assistant', content };
|
||||
yield {
|
||||
type: 'done',
|
||||
message,
|
||||
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
|
||||
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
|
||||
const base = normalizeCompatBase(o.baseUrl);
|
||||
let res: unknown;
|
||||
try {
|
||||
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
|
||||
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
|
||||
const found: DiscoveredModel[] = [];
|
||||
for (const m of list) {
|
||||
if (!isRecord(m)) continue;
|
||||
const id = str(m.id) || str(m.name);
|
||||
if (!id || !isChatModelId(id)) continue;
|
||||
const model: DiscoveredModel = { id };
|
||||
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
|
||||
// OpenRouter-style metadata, when present.
|
||||
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
|
||||
if (ctxLen) model.contextWindow = ctxLen;
|
||||
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
|
||||
if (isRecord(m.pricing)) {
|
||||
const pin = Number(m.pricing.prompt);
|
||||
const pout = Number(m.pricing.completion);
|
||||
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
|
||||
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
|
||||
}
|
||||
const caps: Partial<ModelCapabilities> = {};
|
||||
if (Array.isArray(m.supported_parameters)) {
|
||||
caps.tools = m.supported_parameters.includes('tools');
|
||||
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
|
||||
}
|
||||
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
|
||||
caps.vision = m.architecture.input_modalities.includes('image');
|
||||
}
|
||||
if (Object.keys(caps).length > 0) model.capabilities = caps;
|
||||
found.push(model);
|
||||
}
|
||||
return withCatalogDefaults(o.kind, found);
|
||||
}
|
||||
|
||||
function options(cfg: ProviderConfig): CompatOptions {
|
||||
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
|
||||
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
|
||||
}
|
||||
|
||||
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
|
||||
return runTest(cfg.name || 'the server', async () => {
|
||||
const o = options(cfg);
|
||||
try {
|
||||
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
|
||||
} catch (err) {
|
||||
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
|
||||
// No model listing: prove the connection with a one-token completion instead.
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
|
||||
try {
|
||||
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
|
||||
if (ev.type === 'done') break;
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export const openaiCompatibleAdapter: ProviderAdapter = {
|
||||
kind: 'openai_compatible',
|
||||
meta: {
|
||||
label: 'OpenAI-compatible',
|
||||
defaultBaseUrl: null,
|
||||
requiresApiKey: false,
|
||||
requiresBaseUrl: true,
|
||||
supportsDiscovery: true,
|
||||
docsUrl: 'https://openrouter.ai/docs/quickstart',
|
||||
keyHint: 'Optional for local servers',
|
||||
},
|
||||
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
|
||||
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
|
||||
test,
|
||||
};
|
||||
932
server/ai/providers/providers.test.ts
Normal file
932
server/ai/providers/providers.test.ts
Normal file
|
|
@ -0,0 +1,932 @@
|
|||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
/**
|
||||
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
|
||||
* Wire shapes below are derived from what each adapter's parser reads.
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
|
||||
import { anthropicAdapter } from './anthropic.ts';
|
||||
import { lookupModelDefaults } from './catalog.ts';
|
||||
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
|
||||
import { parseSSE } from './http.ts';
|
||||
import { ollamaAdapter } from './ollama.ts';
|
||||
import { openaiAdapter } from './openai.ts';
|
||||
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const enc = new TextEncoder();
|
||||
|
||||
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream({
|
||||
start(c) {
|
||||
for (const ch of chunks) c.enqueue(ch);
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
|
||||
|
||||
/** One SSE event. */
|
||||
function ev(data: unknown, event?: string): string {
|
||||
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
|
||||
}
|
||||
const DONE = 'data: [DONE]\n\n';
|
||||
|
||||
function rechunk(s: string, n: number): string[] {
|
||||
const out: string[] = [];
|
||||
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** SSE response whose chunk boundaries deliberately fall mid-line. */
|
||||
function sseResponse(chunks: string[], status = 200): Response {
|
||||
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
|
||||
}
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
|
||||
}
|
||||
|
||||
interface Call {
|
||||
url: string;
|
||||
method?: string;
|
||||
headers: Record<string, string>;
|
||||
body: any;
|
||||
}
|
||||
let calls: Call[] = [];
|
||||
type Reply = Response | Error;
|
||||
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
|
||||
|
||||
function stubFetch(handler: Handler): void {
|
||||
calls = [];
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn(async (input: unknown, init?: RequestInit) => {
|
||||
const call: Call = {
|
||||
url: String(input),
|
||||
method: init?.method,
|
||||
headers: (init?.headers ?? {}) as Record<string, string>,
|
||||
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
|
||||
};
|
||||
calls.push(call);
|
||||
const r = await handler(call, calls.length - 1);
|
||||
if (r instanceof Error) throw r;
|
||||
return r;
|
||||
}),
|
||||
);
|
||||
}
|
||||
/** Replies with the given thunks in order (the last one repeats). */
|
||||
function queue(...items: Array<() => Reply>): Handler {
|
||||
return (_c, n) => items[Math.min(n, items.length - 1)]!();
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
|
||||
const out: T[] = [];
|
||||
for await (const x of it) out.push(x);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
|
||||
const events = await drain(adapter.stream(cfg, req));
|
||||
const done = events[events.length - 1];
|
||||
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
|
||||
return { events, done };
|
||||
}
|
||||
|
||||
async function failure(p: Promise<unknown>): Promise<AIError> {
|
||||
const err = await p.then(
|
||||
() => undefined,
|
||||
(e: unknown) => e,
|
||||
);
|
||||
expect(err).toBeInstanceOf(AIError);
|
||||
return err as AIError;
|
||||
}
|
||||
|
||||
const TOOL: ToolDef = {
|
||||
name: 'lookup',
|
||||
description: 'Look things up',
|
||||
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
|
||||
};
|
||||
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
|
||||
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
|
||||
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
|
||||
role: 'assistant',
|
||||
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
|
||||
...extra,
|
||||
});
|
||||
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
|
||||
|
||||
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-adapter fixtures */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
|
||||
|
||||
interface Kit {
|
||||
name: string;
|
||||
adapter: ProviderAdapter;
|
||||
key: string;
|
||||
cfg: ProviderConfig;
|
||||
model: string;
|
||||
otherKind: ProviderKind;
|
||||
streamUrl: string;
|
||||
toolId: unknown;
|
||||
assertAuth(h: Record<string, string>): void;
|
||||
assertRequest(body: any): void; // system prompt + tool definitions
|
||||
assertHistory(body: any): void; // tool_call + tool_result mapping
|
||||
text(parts: string[]): string[];
|
||||
tool(frags: string[]): string[];
|
||||
usage(): { chunks: string[]; expected: Usage };
|
||||
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
|
||||
}
|
||||
|
||||
const openaiStyleErrors = (key: string): Kit['errors'] => ({
|
||||
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
|
||||
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
|
||||
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
|
||||
});
|
||||
|
||||
/* ---- Anthropic ---- */
|
||||
const anthropicKit: Kit = {
|
||||
name: 'anthropic',
|
||||
adapter: anthropicAdapter,
|
||||
key: 'sk-ant-TESTKEY-1234567890',
|
||||
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
|
||||
model: 'claude-sonnet-5-5',
|
||||
otherKind: 'openai',
|
||||
streamUrl: 'https://api.anthropic.com/v1/messages',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
|
||||
expect(h['anthropic-version']).toBe('2026-10-21');
|
||||
expect(h.Authorization).toBeUndefined();
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.system).toBe('SYS');
|
||||
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
|
||||
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.messages).toEqual([
|
||||
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
|
||||
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
|
||||
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
|
||||
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
|
||||
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
|
||||
ev({ type: 'message_stop' }, 'message_stop'),
|
||||
],
|
||||
tool: (frags) => [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
|
||||
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
|
||||
ev({ type: 'content_block_stop', index: 0 }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
],
|
||||
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
],
|
||||
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
|
||||
}),
|
||||
errors: {
|
||||
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
|
||||
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
|
||||
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
|
||||
},
|
||||
};
|
||||
|
||||
/* ---- OpenAI (Responses API) ---- */
|
||||
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
|
||||
const openaiKit: Kit = {
|
||||
name: 'openai',
|
||||
adapter: openaiAdapter,
|
||||
key: 'sk-proj-TESTKEY-1234567890',
|
||||
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
|
||||
model: 'gpt-4o',
|
||||
otherKind: 'anthropic',
|
||||
streamUrl: 'https://api.openai.com/v1/responses',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.store).toBe(false);
|
||||
expect(b.instructions).toBe('SYS');
|
||||
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.input).toEqual([
|
||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
||||
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [
|
||||
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
|
||||
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
|
||||
],
|
||||
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
|
||||
tool: (frags) => [
|
||||
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
|
||||
ev({
|
||||
type: 'response.completed',
|
||||
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
|
||||
}),
|
||||
],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ev({ type: 'response.output_text.delta', delta: 'ok' }),
|
||||
ev({
|
||||
type: 'response.completed',
|
||||
response: {
|
||||
output: [oaMessage('ok')],
|
||||
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
|
||||
},
|
||||
}),
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
|
||||
};
|
||||
|
||||
/* ---- Gemini ---- */
|
||||
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
|
||||
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
|
||||
const geminiKit: Kit = {
|
||||
name: 'gemini',
|
||||
adapter: geminiAdapter,
|
||||
key: 'AIzaSyTESTKEY1234567890abcd',
|
||||
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
|
||||
model: 'gemini-3.8-flash',
|
||||
otherKind: 'anthropic',
|
||||
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
|
||||
toolId: expect.stringMatching(/^gem_/),
|
||||
assertAuth(h) {
|
||||
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
|
||||
expect(h.Authorization).toBeUndefined();
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
|
||||
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.contents).toEqual([
|
||||
{ role: 'user', parts: [{ text: 'hi' }] },
|
||||
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
|
||||
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
|
||||
// Gemini delivers a function call whole (args is an object), never as fragments.
|
||||
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: {
|
||||
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
|
||||
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
|
||||
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
|
||||
},
|
||||
};
|
||||
|
||||
/* ---- Chat Completions family ---- */
|
||||
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
|
||||
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
|
||||
return {
|
||||
...over,
|
||||
otherKind: 'gemini',
|
||||
toolId: 'call_1',
|
||||
assertAuth(h) {
|
||||
expect(h.Authorization).toBe(`Bearer ${over.key}`);
|
||||
},
|
||||
assertRequest(b) {
|
||||
expect(b.stream).toBe(true);
|
||||
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
|
||||
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
|
||||
},
|
||||
assertHistory(b) {
|
||||
expect(b.messages).toEqual([
|
||||
{ role: 'system', content: 'SYS' },
|
||||
{ role: 'user', content: 'hi' },
|
||||
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
|
||||
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
|
||||
]);
|
||||
},
|
||||
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
|
||||
tool: (frags) => [
|
||||
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
|
||||
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
|
||||
ccChunk({}, 'tool_calls'),
|
||||
DONE,
|
||||
],
|
||||
usage: () => ({
|
||||
chunks: [
|
||||
ccChunk({ content: 'ok' }, 'stop'),
|
||||
ev({
|
||||
choices: [],
|
||||
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
|
||||
}),
|
||||
DONE,
|
||||
],
|
||||
expected: EXPECTED_FULL_USAGE,
|
||||
}),
|
||||
errors: openaiStyleErrors(over.key),
|
||||
};
|
||||
}
|
||||
const compatKit = chatCompletionsKit({
|
||||
name: 'openai_compatible',
|
||||
adapter: openaiCompatibleAdapter,
|
||||
key: 'sk-or-TESTKEY-1234567890',
|
||||
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
|
||||
model: 'meta/llama-3',
|
||||
streamUrl: 'https://router.example.com/api/v1/chat/completions',
|
||||
});
|
||||
const ollamaKit = chatCompletionsKit({
|
||||
name: 'ollama',
|
||||
adapter: ollamaAdapter,
|
||||
key: 'ollama-proxy-secret-1234',
|
||||
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
|
||||
model: 'llama3',
|
||||
streamUrl: 'http://localhost:11434/v1/chat/completions',
|
||||
});
|
||||
|
||||
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Shared behaviour across all five adapters */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe.each(kits)('$name adapter', (kit) => {
|
||||
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
|
||||
|
||||
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(calls[0]!.url).toBe(kit.streamUrl);
|
||||
expect(calls[0]!.url).not.toContain(kit.key);
|
||||
expect(calls[0]!.method).toBe('POST');
|
||||
kit.assertAuth(calls[0]!.headers);
|
||||
kit.assertRequest(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('maps tool_call and tool_result history into the provider format', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
|
||||
kit.assertHistory(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
|
||||
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
|
||||
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
|
||||
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
|
||||
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
|
||||
kit.assertHistory(calls[0]!.body);
|
||||
});
|
||||
|
||||
it('streams text events in order and concatenates them in done', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
|
||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
|
||||
expect(done.message.role).toBe('assistant');
|
||||
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
|
||||
expect(done.stopReason).toBe('end');
|
||||
});
|
||||
|
||||
it('assembles a fragmented tool call into one tool_call event', async () => {
|
||||
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
|
||||
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
|
||||
expect(toolEvents).toHaveLength(1);
|
||||
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
|
||||
expect(events[events.length - 1]).toBe(done);
|
||||
});
|
||||
|
||||
it('maps usage onto the neutral convention', async () => {
|
||||
const { chunks, expected } = kit.usage();
|
||||
stubFetch(queue(() => sseResponse(chunks)));
|
||||
const { done } = await run(kit.adapter, kit.cfg, baseReq());
|
||||
expect(done.usage).toEqual(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['auth', 'auth', 401],
|
||||
['rate', 'rate_limit', 429],
|
||||
['ctx', 'context_length', 400],
|
||||
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
|
||||
const [st, body] = kit.errors[which];
|
||||
stubFetch(queue(() => jsonResponse(body, st)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.code).toBe(code);
|
||||
expect(err.status).toBe(status);
|
||||
expect(err.message).not.toContain(kit.key);
|
||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
||||
});
|
||||
|
||||
it('scrubs the API key from provider text', async () => {
|
||||
const [st, body] = kit.errors.auth;
|
||||
stubFetch(queue(() => jsonResponse(body, st)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.providerMessage).toContain('[redacted]');
|
||||
expect(JSON.stringify(err)).not.toContain(kit.key);
|
||||
});
|
||||
|
||||
it('maps a fetch TypeError to a network error without leaking the key', async () => {
|
||||
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
|
||||
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
|
||||
expect(err.code).toBe('network');
|
||||
expect(err.retryable).toBe(true);
|
||||
expect(err.message).not.toContain(kit.key);
|
||||
expect(err.providerMessage ?? '').not.toContain(kit.key);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Anthropic specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('anthropic providerData', () => {
|
||||
const thinkingTurn = [
|
||||
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
|
||||
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
|
||||
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
|
||||
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
|
||||
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
|
||||
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
|
||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
|
||||
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
|
||||
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
|
||||
ev({ type: 'message_stop' }),
|
||||
];
|
||||
const replayed = [
|
||||
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
|
||||
{ type: 'text', text: 'Sure' },
|
||||
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
|
||||
];
|
||||
const followUp = (assistant: ChatMessage): ChatRequest => ({
|
||||
model: 'claude-sonnet-5-5',
|
||||
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
|
||||
});
|
||||
|
||||
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
|
||||
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
|
||||
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
|
||||
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
|
||||
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
|
||||
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
|
||||
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
|
||||
|
||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
|
||||
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
|
||||
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
|
||||
});
|
||||
|
||||
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
|
||||
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
|
||||
() => sseResponse(anthropicKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
|
||||
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('anthropic listModels', () => {
|
||||
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() =>
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: 'claude-haiku-4-5',
|
||||
display_name: 'Haiku API Name',
|
||||
max_input_tokens: 200000,
|
||||
max_tokens: 64000,
|
||||
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
|
||||
},
|
||||
{ id: 'claude-embed-1' },
|
||||
],
|
||||
has_more: true,
|
||||
last_id: 'claude-haiku-4-5',
|
||||
}),
|
||||
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
|
||||
),
|
||||
);
|
||||
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
|
||||
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
|
||||
expect(calls[0]!.url).not.toContain(anthropicKit.key);
|
||||
anthropicKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
|
||||
expect(models[0]).toMatchObject({
|
||||
label: 'Haiku API Name', // reported by the API, wins over the catalog
|
||||
contextWindow: 200000,
|
||||
maxOutput: 64000,
|
||||
priceIn: 1, // from the catalog
|
||||
priceOut: 5,
|
||||
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
|
||||
});
|
||||
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('openai providerData and reasoning', () => {
|
||||
const output = [
|
||||
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
|
||||
oaMessage('Hi'),
|
||||
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
|
||||
];
|
||||
const turn = [
|
||||
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
|
||||
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
|
||||
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
|
||||
];
|
||||
|
||||
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
|
||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
|
||||
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
|
||||
const body = calls[0]!.body;
|
||||
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
|
||||
expect(body.include).toEqual(['reasoning.encrypted_content']);
|
||||
expect(body.temperature).toBeUndefined(); // reasoning models reject it
|
||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
|
||||
|
||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
|
||||
expect(calls[1]!.body.input).toEqual([
|
||||
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
|
||||
...output,
|
||||
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('retries once without reasoning.effort when the model rejects it', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
|
||||
() => sseResponse(openaiKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
|
||||
expect(calls[0]!.body.reasoning.effort).toBe('none');
|
||||
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
|
||||
});
|
||||
});
|
||||
|
||||
describe('openai listModels', () => {
|
||||
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
|
||||
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
|
||||
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
|
||||
const models = await openaiAdapter.listModels(openaiKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
|
||||
openaiKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
|
||||
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
|
||||
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Gemini specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('gemini providerData and request config', () => {
|
||||
const turn = [
|
||||
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
|
||||
gemChunk([{ text: 'Hello' }]),
|
||||
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
|
||||
];
|
||||
const replayed = [
|
||||
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
|
||||
{ text: 'Hello' },
|
||||
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
|
||||
];
|
||||
|
||||
it('keeps thought signatures in providerData and replays the raw parts', async () => {
|
||||
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
|
||||
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
|
||||
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
|
||||
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
|
||||
expect(done.stopReason).toBe('tool_use');
|
||||
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
|
||||
|
||||
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
|
||||
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
|
||||
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
|
||||
});
|
||||
|
||||
it('turns a blocked prompt into a content_filter error', async () => {
|
||||
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
|
||||
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
|
||||
expect(err.code).toBe('content_filter');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gemini listModels', () => {
|
||||
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() =>
|
||||
jsonResponse({
|
||||
models: [
|
||||
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
|
||||
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
|
||||
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
|
||||
],
|
||||
nextPageToken: 'tok1',
|
||||
}),
|
||||
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
|
||||
),
|
||||
);
|
||||
const models = await geminiAdapter.listModels(geminiKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
|
||||
expect(calls[1]!.url).toContain('pageToken=tok1');
|
||||
for (const c of calls) {
|
||||
expect(c.url).not.toContain(geminiKit.key);
|
||||
geminiKit.assertAuth(c.headers);
|
||||
}
|
||||
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
|
||||
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
|
||||
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('toGeminiSchema', () => {
|
||||
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
|
||||
const out = toGeminiSchema({
|
||||
$schema: 'http://json-schema.org/draft-07/schema#',
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
|
||||
when: { type: 'string', format: 'date-time' },
|
||||
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
|
||||
mode: { enum: ['a', 'b'] },
|
||||
nested: { type: 'object' },
|
||||
},
|
||||
required: ['name'],
|
||||
});
|
||||
expect(out).toEqual({
|
||||
type: 'object',
|
||||
properties: {
|
||||
name: { type: 'string', nullable: true },
|
||||
when: { type: 'string', format: 'date-time' },
|
||||
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
|
||||
mode: { enum: ['a', 'b'], type: 'string' },
|
||||
nested: { type: 'object', properties: {} },
|
||||
},
|
||||
required: ['name'],
|
||||
});
|
||||
});
|
||||
it('falls back to a string schema for non-objects', () => {
|
||||
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* OpenAI-compatible and Ollama specifics */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('openai-compatible stream_options fallback', () => {
|
||||
it('retries without stream_options when the first 400 complains about it', async () => {
|
||||
stubFetch(
|
||||
queue(
|
||||
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
|
||||
() => sseResponse(compatKit.text(['ok'])),
|
||||
),
|
||||
);
|
||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
|
||||
expect(calls[1]!.body.stream_options).toBeUndefined();
|
||||
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
|
||||
});
|
||||
|
||||
it('does not write providerData (it only speaks neutral content)', async () => {
|
||||
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
|
||||
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
|
||||
expect(done.message.providerData).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('openai-compatible listModels', () => {
|
||||
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
|
||||
stubFetch(
|
||||
queue(() =>
|
||||
jsonResponse({
|
||||
data: [
|
||||
{
|
||||
id: 'anthropic/claude-sonnet-5-5',
|
||||
name: 'Claude Sonnet 5.5',
|
||||
context_length: 1000000,
|
||||
top_provider: { max_completion_tokens: 64000 },
|
||||
pricing: { prompt: '0.000002', completion: '0.00001' },
|
||||
supported_parameters: ['tools', 'reasoning'],
|
||||
architecture: { input_modalities: ['text', 'image'] },
|
||||
},
|
||||
{ id: 'openai/text-embedding-3-small' },
|
||||
{ id: 'vendor/plain' },
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
|
||||
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
|
||||
compatKit.assertAuth(calls[0]!.headers);
|
||||
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
|
||||
expect(models[0]).toMatchObject({
|
||||
label: 'Claude Sonnet 5.5',
|
||||
contextWindow: 1000000,
|
||||
maxOutput: 64000,
|
||||
priceIn: 2,
|
||||
priceOut: 10,
|
||||
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
|
||||
});
|
||||
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
|
||||
expect(models[1]).toEqual({ id: 'vendor/plain' });
|
||||
});
|
||||
|
||||
it('reports "unsupported" when the server has no /models endpoint', async () => {
|
||||
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
|
||||
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
|
||||
expect(err.code).toBe('unsupported');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ollama listModels', () => {
|
||||
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
|
||||
stubFetch((c) => {
|
||||
if (c.url.endsWith('/api/tags')) {
|
||||
return jsonResponse({
|
||||
models: [
|
||||
{ name: 'old:1b', model: 'old:1b' },
|
||||
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
|
||||
{ name: 'llama3:latest', model: 'llama3:latest' },
|
||||
],
|
||||
});
|
||||
}
|
||||
if (c.url.endsWith('/api/show')) {
|
||||
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
|
||||
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
|
||||
return jsonResponse({ error: 'unsupported' }, 404);
|
||||
}
|
||||
return new Response('?', { status: 404 });
|
||||
});
|
||||
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
|
||||
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
|
||||
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
|
||||
ollamaKit.assertAuth(calls[0]!.headers);
|
||||
expect(models).toEqual([
|
||||
{
|
||||
id: 'llama3:latest',
|
||||
label: 'llama3:latest',
|
||||
priceIn: 0,
|
||||
priceOut: 0,
|
||||
contextWindow: 8192,
|
||||
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
|
||||
},
|
||||
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('strips /v1 from a configured base URL', async () => {
|
||||
stubFetch(queue(() => jsonResponse({ models: [] })));
|
||||
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
|
||||
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeCompatBase', () => {
|
||||
it.each([
|
||||
['https://api.x.com/v1', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/', 'https://api.x.com/v1'],
|
||||
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
|
||||
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
|
||||
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
|
||||
['http://localhost:1234', 'http://localhost:1234/v1'],
|
||||
['http://localhost:1234/', 'http://localhost:1234/v1'],
|
||||
])('%s -> %s', (input, expected) => {
|
||||
expect(normalizeCompatBase(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* catalog */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('lookupModelDefaults', () => {
|
||||
it('matches exact ids', () => {
|
||||
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
|
||||
});
|
||||
it('matches dated snapshots by prefix, without the base label', () => {
|
||||
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
|
||||
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
|
||||
expect(d.label).toBeUndefined();
|
||||
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
|
||||
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
|
||||
});
|
||||
it('keeps capabilities but drops prices for a different variant of a known model', () => {
|
||||
const d = lookupModelDefaults('openai', 'o3-deep-research');
|
||||
expect(d.capabilities?.reasoning).toBe(true);
|
||||
expect(d.priceIn).toBeUndefined();
|
||||
expect(d.priceOut).toBeUndefined();
|
||||
});
|
||||
it('returns {} for unknown models and for kinds without a table', () => {
|
||||
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
|
||||
expect(lookupModelDefaults('anthropic', '')).toEqual({});
|
||||
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
|
||||
});
|
||||
it('borrows capabilities but not prices or temperature for router ids', () => {
|
||||
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
|
||||
expect(d.priceIn).toBeUndefined();
|
||||
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
|
||||
expect(d.capabilities).not.toHaveProperty('temperature');
|
||||
});
|
||||
it('returns copies that are safe to mutate', () => {
|
||||
const a = lookupModelDefaults('openai', 'gpt-4o');
|
||||
a.capabilities!.tools = false;
|
||||
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* parseSSE */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
describe('parseSSE', () => {
|
||||
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
|
||||
|
||||
it('handles CRLF line endings', async () => {
|
||||
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
|
||||
});
|
||||
it('joins multi-line data with newlines', async () => {
|
||||
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
|
||||
});
|
||||
it('ignores comments and keep-alives', async () => {
|
||||
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
|
||||
});
|
||||
it('stops at [DONE]', async () => {
|
||||
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
|
||||
});
|
||||
it('handles chunk boundaries inside a line', async () => {
|
||||
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
|
||||
});
|
||||
it('handles CRLF split across chunks', async () => {
|
||||
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
||||
});
|
||||
it('handles bare CR line endings', async () => {
|
||||
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
|
||||
});
|
||||
it('flushes a final event with no trailing blank line', async () => {
|
||||
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
|
||||
});
|
||||
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
|
||||
const bytes = enc.encode('data: €uro\n\n');
|
||||
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
|
||||
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
|
||||
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
|
||||
});
|
||||
});
|
||||
624
server/ai/service.ts
Normal file
624
server/ai/service.ts
Normal file
|
|
@ -0,0 +1,624 @@
|
|||
/**
|
||||
* Provider + model configuration, task routing, usage accounting and the two call
|
||||
* primitives everything else builds on: `runChat` (one streamed model turn) and
|
||||
* `generateStructured` (schema-constrained JSON through a forced tool call).
|
||||
*/
|
||||
import { jsonrepair } from 'jsonrepair';
|
||||
import { z } from 'zod';
|
||||
import { decryptSecret, encryptSecret, maskSecret } from '../crypto.ts';
|
||||
import type { DB } from '../db.ts';
|
||||
import { HttpError, notFound, type AppContext } from '../http.ts';
|
||||
import { getSetting, setSetting } from '../settings.ts';
|
||||
import { newId, now, parseJson } from '../util.ts';
|
||||
import { getAdapter, lookupModelDefaults } from './providers/index.ts';
|
||||
import {
|
||||
AIError,
|
||||
DEFAULT_CAPABILITIES,
|
||||
PROVIDER_KINDS,
|
||||
type ChatMessage,
|
||||
type ChatRequest,
|
||||
type DiscoveredModel,
|
||||
type ModelCapabilities,
|
||||
type ProviderConfig,
|
||||
type ProviderKind,
|
||||
type ReasoningEffort,
|
||||
type StopReason,
|
||||
type StreamEvent,
|
||||
type ToolDef,
|
||||
type Usage,
|
||||
} from './types.ts';
|
||||
|
||||
/* ------------------------------ rows + DTOs ------------------------------ */
|
||||
|
||||
interface ProviderRow {
|
||||
id: string;
|
||||
name: string;
|
||||
kind: ProviderKind;
|
||||
base_url: string | null;
|
||||
api_key_enc: string | null;
|
||||
headers_enc: string | null;
|
||||
enabled: number;
|
||||
last_test_at: number | null;
|
||||
last_test_ok: number | null;
|
||||
last_test_message: string;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
interface ModelRow {
|
||||
id: string;
|
||||
provider_id: string;
|
||||
model: string;
|
||||
label: string;
|
||||
source: string;
|
||||
enabled: number;
|
||||
capabilities: string;
|
||||
context_window: number | null;
|
||||
max_output: number | null;
|
||||
price_in: number | null;
|
||||
price_out: number | null;
|
||||
}
|
||||
|
||||
export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const;
|
||||
export type TaskName = (typeof TASKS)[number];
|
||||
|
||||
export const TASK_INFO: Record<TaskName, { label: string; description: string }> = {
|
||||
chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' },
|
||||
generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' },
|
||||
vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' },
|
||||
quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' },
|
||||
};
|
||||
|
||||
export interface TaskConfig {
|
||||
modelId: string | null;
|
||||
temperature: number | null;
|
||||
reasoningEffort: ReasoningEffort | null;
|
||||
maxOutputTokens: number | null;
|
||||
}
|
||||
export type TaskSettings = Record<TaskName | 'default', TaskConfig>;
|
||||
|
||||
const EMPTY_TASK: TaskConfig = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
|
||||
|
||||
export const taskConfigSchema = z.object({
|
||||
modelId: z.string().nullable(),
|
||||
temperature: z.number().min(0).max(2).nullable(),
|
||||
reasoningEffort: z.enum(['none', 'low', 'medium', 'high']).nullable(),
|
||||
maxOutputTokens: z.number().int().min(256).max(200000).nullable(),
|
||||
});
|
||||
|
||||
function mapProvider(app: AppContext, row: ProviderRow) {
|
||||
const key = decryptSecret(row.api_key_enc, app.config.secretKey);
|
||||
const headers = parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {});
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
kind: row.kind,
|
||||
baseUrl: row.base_url,
|
||||
hasKey: Boolean(row.api_key_enc),
|
||||
/** Only ever a masked preview; the key itself never leaves the server. */
|
||||
keyPreview: key ? maskSecret(key) : row.api_key_enc ? 'unreadable (re-enter key)' : '',
|
||||
headerNames: Object.keys(headers),
|
||||
enabled: Boolean(row.enabled),
|
||||
lastTestAt: row.last_test_at,
|
||||
lastTestOk: row.last_test_ok === null ? null : Boolean(row.last_test_ok),
|
||||
lastTestMessage: row.last_test_message,
|
||||
createdAt: row.created_at,
|
||||
};
|
||||
}
|
||||
|
||||
function mapModel(row: ModelRow & { provider_name?: string; provider_kind?: string; provider_enabled?: number }) {
|
||||
return {
|
||||
id: row.id,
|
||||
providerId: row.provider_id,
|
||||
providerName: row.provider_name ?? '',
|
||||
providerKind: (row.provider_kind ?? '') as ProviderKind,
|
||||
model: row.model,
|
||||
label: row.label || row.model,
|
||||
source: row.source,
|
||||
enabled: Boolean(row.enabled),
|
||||
available: Boolean(row.enabled) && row.provider_enabled !== 0,
|
||||
capabilities: { ...DEFAULT_CAPABILITIES, ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}) },
|
||||
contextWindow: row.context_window,
|
||||
maxOutput: row.max_output,
|
||||
priceIn: row.price_in,
|
||||
priceOut: row.price_out,
|
||||
};
|
||||
}
|
||||
export type ModelDto = ReturnType<typeof mapModel>;
|
||||
|
||||
const MODEL_SELECT = `SELECT m.*, p.name AS provider_name, p.kind AS provider_kind, p.enabled AS provider_enabled FROM ai_models m JOIN ai_providers p ON p.id = m.provider_id`;
|
||||
|
||||
/* ------------------------------ providers ------------------------------ */
|
||||
|
||||
export const providerInputSchema = z.object({
|
||||
name: z.string().trim().min(1).max(80),
|
||||
kind: z.enum(PROVIDER_KINDS as [ProviderKind, ...ProviderKind[]]),
|
||||
baseUrl: z
|
||||
.string()
|
||||
.trim()
|
||||
.max(500)
|
||||
.nullish()
|
||||
.transform((v) => v || null)
|
||||
.refine((v) => v === null || /^https?:\/\//i.test(v), 'URL must start with http:// or https://'),
|
||||
/** undefined = keep the stored key, '' or null = remove it. */
|
||||
apiKey: z.string().max(2000).nullish(),
|
||||
headers: z.record(z.string().max(100), z.string().max(2000)).optional(),
|
||||
enabled: z.boolean().optional(),
|
||||
});
|
||||
|
||||
function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig {
|
||||
return {
|
||||
id: row.id,
|
||||
kind: row.kind,
|
||||
name: row.name,
|
||||
baseUrl: row.base_url,
|
||||
apiKey: decryptSecret(row.api_key_enc, app.config.secretKey),
|
||||
headers: parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {}),
|
||||
};
|
||||
}
|
||||
|
||||
function providerRow(db: DB, id: string): ProviderRow {
|
||||
const row = db.get<ProviderRow>('SELECT * FROM ai_providers WHERE id = ?', id);
|
||||
if (!row) throw notFound('Provider');
|
||||
return row;
|
||||
}
|
||||
|
||||
export function listProviders(app: AppContext) {
|
||||
return app.db.all<ProviderRow>('SELECT * FROM ai_providers ORDER BY created_at').map((r) => mapProvider(app, r));
|
||||
}
|
||||
|
||||
export function saveProvider(app: AppContext, id: string | null, input: z.input<typeof providerInputSchema>) {
|
||||
const data = providerInputSchema.parse(input);
|
||||
const meta = getAdapter(data.kind).meta;
|
||||
if (meta.requiresBaseUrl && !data.baseUrl) throw new HttpError(422, 'validation', 'This provider type needs a server URL');
|
||||
const at = now();
|
||||
const key = app.config.secretKey;
|
||||
if (id) {
|
||||
const row = providerRow(app.db, id);
|
||||
const apiKeyEnc = data.apiKey === undefined ? row.api_key_enc : data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null;
|
||||
const headersEnc = data.headers === undefined ? row.headers_enc : Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null;
|
||||
app.db.run(
|
||||
'UPDATE ai_providers SET name = ?, kind = ?, base_url = ?, api_key_enc = ?, headers_enc = ?, enabled = ?, updated_at = ? WHERE id = ?',
|
||||
data.name,
|
||||
data.kind,
|
||||
data.baseUrl,
|
||||
apiKeyEnc,
|
||||
headersEnc,
|
||||
data.enabled === undefined ? row.enabled : data.enabled ? 1 : 0,
|
||||
at,
|
||||
id,
|
||||
);
|
||||
return mapProvider(app, providerRow(app.db, id));
|
||||
}
|
||||
const newProviderId = newId();
|
||||
app.db.run(
|
||||
'INSERT INTO ai_providers (id, name, kind, base_url, api_key_enc, headers_enc, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
|
||||
newProviderId,
|
||||
data.name,
|
||||
data.kind,
|
||||
data.baseUrl,
|
||||
data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null,
|
||||
data.headers && Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null,
|
||||
data.enabled === false ? 0 : 1,
|
||||
at,
|
||||
at,
|
||||
);
|
||||
return mapProvider(app, providerRow(app.db, newProviderId));
|
||||
}
|
||||
|
||||
export function deleteProvider(app: AppContext, id: string): void {
|
||||
providerRow(app.db, id);
|
||||
const modelIds = new Set(app.db.all<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ?', id).map((m) => m.id));
|
||||
app.db.run('DELETE FROM ai_providers WHERE id = ?', id);
|
||||
const tasks = getTaskSettings(app.db);
|
||||
let changed = false;
|
||||
for (const name of Object.keys(tasks) as (keyof TaskSettings)[]) {
|
||||
if (tasks[name].modelId && modelIds.has(tasks[name].modelId!)) {
|
||||
tasks[name].modelId = null;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) setSetting(app.db, 'ai.tasks', tasks);
|
||||
}
|
||||
|
||||
export async function testProvider(app: AppContext, id: string) {
|
||||
const row = providerRow(app.db, id);
|
||||
const anyModel = app.db.get<{ model: string }>('SELECT model FROM ai_models WHERE provider_id = ? ORDER BY enabled DESC LIMIT 1', id)?.model;
|
||||
const result = await getAdapter(row.kind).test(providerConfig(app, row), anyModel);
|
||||
app.db.run('UPDATE ai_providers SET last_test_at = ?, last_test_ok = ?, last_test_message = ? WHERE id = ?', now(), result.ok ? 1 : 0, result.message.slice(0, 500), id);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Looks up the provider's model list and merges it into ai_models without disturbing user edits. */
|
||||
export async function discoverModels(app: AppContext, id: string): Promise<{ added: number; total: number }> {
|
||||
const row = providerRow(app.db, id);
|
||||
const found = await getAdapter(row.kind).listModels(providerConfig(app, row));
|
||||
const existing = new Map(app.db.all<ModelRow>('SELECT * FROM ai_models WHERE provider_id = ?', id).map((m) => [m.model, m]));
|
||||
const at = now();
|
||||
let added = 0;
|
||||
app.db.tx(() => {
|
||||
for (const m of found) {
|
||||
const current = existing.get(m.id);
|
||||
if (current) {
|
||||
// Refresh facts the provider reports, but never overwrite prices or flags the user may have edited.
|
||||
app.db.run(
|
||||
'UPDATE ai_models SET context_window = COALESCE(?, context_window), max_output = COALESCE(?, max_output), price_in = COALESCE(price_in, ?), price_out = COALESCE(price_out, ?), updated_at = ? WHERE id = ?',
|
||||
m.contextWindow ?? null,
|
||||
m.maxOutput ?? null,
|
||||
m.priceIn ?? null,
|
||||
m.priceOut ?? null,
|
||||
at,
|
||||
current.id,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
insertModel(app.db, id, m, 'discovered', false);
|
||||
added += 1;
|
||||
}
|
||||
});
|
||||
return { added, total: found.length };
|
||||
}
|
||||
|
||||
function insertModel(db: DB, providerId: string, m: DiscoveredModel, source: string, enabled: boolean): string {
|
||||
const id = newId();
|
||||
const at = now();
|
||||
db.run(
|
||||
`INSERT INTO ai_models (id, provider_id, model, label, source, enabled, capabilities, context_window, max_output, price_in, price_out, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
id,
|
||||
providerId,
|
||||
m.id,
|
||||
m.label ?? '',
|
||||
source,
|
||||
enabled ? 1 : 0,
|
||||
JSON.stringify(m.capabilities ?? {}),
|
||||
m.contextWindow ?? null,
|
||||
m.maxOutput ?? null,
|
||||
m.priceIn ?? null,
|
||||
m.priceOut ?? null,
|
||||
at,
|
||||
at,
|
||||
);
|
||||
return id;
|
||||
}
|
||||
|
||||
/* ------------------------------ models ------------------------------ */
|
||||
|
||||
export function listModels(db: DB): ModelDto[] {
|
||||
return db.all<ModelRow>(`${MODEL_SELECT} ORDER BY p.created_at, m.enabled DESC, m.model`).map(mapModel);
|
||||
}
|
||||
|
||||
export function getModel(db: DB, id: string): ModelDto {
|
||||
const row = db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ?`, id);
|
||||
if (!row) throw notFound('Model');
|
||||
return mapModel(row);
|
||||
}
|
||||
|
||||
export const manualModelSchema = z.object({
|
||||
providerId: z.string(),
|
||||
model: z.string().trim().min(1).max(200),
|
||||
label: z.string().trim().max(120).optional(),
|
||||
});
|
||||
|
||||
export function addManualModel(app: AppContext, input: z.input<typeof manualModelSchema>): ModelDto {
|
||||
const data = manualModelSchema.parse(input);
|
||||
const provider = providerRow(app.db, data.providerId);
|
||||
const existing = app.db.get<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ? AND model = ?', provider.id, data.model);
|
||||
if (existing) {
|
||||
app.db.run('UPDATE ai_models SET enabled = 1, updated_at = ? WHERE id = ?', now(), existing.id);
|
||||
return getModel(app.db, existing.id);
|
||||
}
|
||||
const defaults = lookupModelDefaults(provider.kind, data.model);
|
||||
const id = insertModel(app.db, provider.id, { ...defaults, id: data.model, label: data.label || defaults.label }, 'manual', true);
|
||||
return getModel(app.db, id);
|
||||
}
|
||||
|
||||
export const modelPatchSchema = z.object({
|
||||
enabled: z.boolean().optional(),
|
||||
label: z.string().trim().max(120).optional(),
|
||||
capabilities: z
|
||||
.object({ tools: z.boolean(), vision: z.boolean(), pdf: z.boolean(), reasoning: z.boolean(), temperature: z.boolean() })
|
||||
.partial()
|
||||
.optional(),
|
||||
contextWindow: z.number().int().min(256).max(100_000_000).nullable().optional(),
|
||||
maxOutput: z.number().int().min(16).max(10_000_000).nullable().optional(),
|
||||
priceIn: z.number().min(0).max(100000).nullable().optional(),
|
||||
priceOut: z.number().min(0).max(100000).nullable().optional(),
|
||||
});
|
||||
|
||||
export function patchModel(db: DB, id: string, input: z.input<typeof modelPatchSchema>): ModelDto {
|
||||
const patch = modelPatchSchema.parse(input);
|
||||
const row = db.get<ModelRow>('SELECT * FROM ai_models WHERE id = ?', id);
|
||||
if (!row) throw notFound('Model');
|
||||
const caps = { ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}), ...patch.capabilities };
|
||||
db.run(
|
||||
'UPDATE ai_models SET enabled = ?, label = ?, capabilities = ?, context_window = ?, max_output = ?, price_in = ?, price_out = ?, updated_at = ? WHERE id = ?',
|
||||
patch.enabled === undefined ? row.enabled : patch.enabled ? 1 : 0,
|
||||
patch.label ?? row.label,
|
||||
JSON.stringify(caps),
|
||||
patch.contextWindow === undefined ? row.context_window : patch.contextWindow,
|
||||
patch.maxOutput === undefined ? row.max_output : patch.maxOutput,
|
||||
patch.priceIn === undefined ? row.price_in : patch.priceIn,
|
||||
patch.priceOut === undefined ? row.price_out : patch.priceOut,
|
||||
now(),
|
||||
id,
|
||||
);
|
||||
return getModel(db, id);
|
||||
}
|
||||
|
||||
export function deleteModel(db: DB, id: string): void {
|
||||
const { changes } = db.run('DELETE FROM ai_models WHERE id = ?', id);
|
||||
if (!changes) throw notFound('Model');
|
||||
}
|
||||
|
||||
/* ------------------------------ task routing ------------------------------ */
|
||||
|
||||
export function getTaskSettings(db: DB): TaskSettings {
|
||||
const stored = getSetting<Partial<TaskSettings>>(db, 'ai.tasks', {});
|
||||
const out = { default: { ...EMPTY_TASK, ...stored.default } } as TaskSettings;
|
||||
for (const t of TASKS) out[t] = { ...EMPTY_TASK, ...stored[t] };
|
||||
return out;
|
||||
}
|
||||
|
||||
export function saveTaskSettings(db: DB, input: unknown): TaskSettings {
|
||||
const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial();
|
||||
const patch = schema.parse(input);
|
||||
const next = { ...getTaskSettings(db), ...patch } as TaskSettings;
|
||||
setSetting(db, 'ai.tasks', next);
|
||||
return next;
|
||||
}
|
||||
|
||||
export interface ResolvedModel {
|
||||
provider: ProviderConfig;
|
||||
model: ModelDto;
|
||||
params: { temperature?: number; reasoningEffort?: ReasoningEffort; maxOutputTokens?: number };
|
||||
}
|
||||
|
||||
/**
|
||||
* Picks the model for a task: explicit override, then the task's own model, then the
|
||||
* default model, then (so a fresh install works after adding one model) any enabled model.
|
||||
*/
|
||||
export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel {
|
||||
const tasks = getTaskSettings(app.db);
|
||||
const cfg = tasks[task];
|
||||
const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x));
|
||||
let model: ModelDto | undefined;
|
||||
for (const id of candidates) {
|
||||
const row = app.db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id);
|
||||
if (row) {
|
||||
model = mapModel(row);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!model) {
|
||||
const needVision = task === 'vision';
|
||||
const rows = app.db.all<ModelRow>(`${MODEL_SELECT} WHERE m.enabled = 1 AND p.enabled = 1 ORDER BY p.created_at, m.created_at`).map(mapModel);
|
||||
model = needVision ? rows.find((m) => m.capabilities.vision) : rows[0];
|
||||
}
|
||||
if (!model) {
|
||||
throw new AIError(
|
||||
'not_configured',
|
||||
task === 'vision' ? 'No vision-capable AI model is set up. Add one in Settings, then choose it for "Vision and OCR".' : 'No AI model is set up yet. Add a provider and enable a model in Settings.',
|
||||
);
|
||||
}
|
||||
const provider = providerConfig(app, providerRow(app.db, model.providerId));
|
||||
const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK;
|
||||
const fallback = tasks.default;
|
||||
const params: ResolvedModel['params'] = {};
|
||||
const temperature = own.temperature ?? fallback.temperature;
|
||||
const effort = own.reasoningEffort ?? fallback.reasoningEffort;
|
||||
const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens;
|
||||
if (temperature !== null && model.capabilities.temperature) params.temperature = temperature;
|
||||
if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort;
|
||||
if (maxOut !== null) params.maxOutputTokens = maxOut;
|
||||
return { provider, model, params };
|
||||
}
|
||||
|
||||
/* ------------------------------ usage ------------------------------ */
|
||||
|
||||
export function estimateCost(model: Pick<ModelDto, 'priceIn' | 'priceOut'>, usage: Usage): number | null {
|
||||
if (model.priceIn === null && model.priceOut === null) return null;
|
||||
// Cached input is billed at a fraction of the list price by every provider; 10% is a close, conservative stand-in.
|
||||
const cached = usage.cacheReadTokens ?? 0;
|
||||
const fresh = Math.max(0, usage.inputTokens - cached);
|
||||
return ((fresh + cached * 0.1) * (model.priceIn ?? 0) + usage.outputTokens * (model.priceOut ?? 0)) / 1_000_000;
|
||||
}
|
||||
|
||||
function recordUsage(app: AppContext, r: ResolvedModel, task: string, usage: Usage | null, started: number, error?: string): void {
|
||||
try {
|
||||
app.db.run(
|
||||
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
now(),
|
||||
r.provider.id,
|
||||
r.provider.name,
|
||||
r.model.model,
|
||||
task,
|
||||
usage?.inputTokens ?? 0,
|
||||
usage?.outputTokens ?? 0,
|
||||
usage?.cacheReadTokens ?? 0,
|
||||
usage ? estimateCost(r.model, usage) : null,
|
||||
error ? 0 : 1,
|
||||
(error ?? '').slice(0, 300),
|
||||
now() - started,
|
||||
);
|
||||
} catch {
|
||||
/* accounting must never break a request */
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------ call primitives ------------------------------ */
|
||||
|
||||
export interface ChatOptions {
|
||||
task: TaskName;
|
||||
/** Label stored with usage rows ("assistant", "quiz generation"...). Defaults to the task. */
|
||||
purpose?: string;
|
||||
modelId?: string | null;
|
||||
system?: string;
|
||||
messages: ChatMessage[];
|
||||
tools?: ToolDef[];
|
||||
toolChoice?: ChatRequest['toolChoice'];
|
||||
jsonMode?: boolean;
|
||||
temperature?: number;
|
||||
reasoningEffort?: ReasoningEffort;
|
||||
maxOutputTokens?: number;
|
||||
signal?: AbortSignal;
|
||||
onEvent?: (event: StreamEvent) => void;
|
||||
}
|
||||
|
||||
export interface ChatResult {
|
||||
message: ChatMessage;
|
||||
text: string;
|
||||
stopReason: StopReason;
|
||||
usage: Usage;
|
||||
model: ModelDto;
|
||||
}
|
||||
|
||||
const sleep = (ms: number, signal?: AbortSignal) =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
const t = setTimeout(resolve, ms);
|
||||
signal?.addEventListener('abort', () => {
|
||||
clearTimeout(t);
|
||||
reject(new AIError('aborted', 'The request was cancelled.'));
|
||||
});
|
||||
});
|
||||
|
||||
/** Runs one model turn. Retries transient failures, but only while nothing has been streamed yet. */
|
||||
export async function runChat(app: AppContext, opts: ChatOptions): Promise<ChatResult> {
|
||||
const resolved = resolveModel(app, opts.task, opts.modelId);
|
||||
const caps = resolved.model.capabilities;
|
||||
const req: ChatRequest = {
|
||||
model: resolved.model.model,
|
||||
system: opts.system,
|
||||
messages: opts.messages,
|
||||
tools: opts.tools,
|
||||
toolChoice: opts.toolChoice,
|
||||
jsonMode: opts.jsonMode,
|
||||
capabilities: caps,
|
||||
signal: opts.signal,
|
||||
temperature: opts.temperature ?? resolved.params.temperature,
|
||||
reasoningEffort: opts.reasoningEffort ?? resolved.params.reasoningEffort,
|
||||
maxOutputTokens: opts.maxOutputTokens ?? resolved.params.maxOutputTokens ?? undefined,
|
||||
};
|
||||
if (req.maxOutputTokens && resolved.model.maxOutput) req.maxOutputTokens = Math.min(req.maxOutputTokens, resolved.model.maxOutput);
|
||||
if (opts.tools?.length && !caps.tools && (opts.toolChoice === 'required' || typeof opts.toolChoice === 'object')) {
|
||||
throw new AIError('unsupported', `${resolved.model.label} is marked as not supporting tools. Choose another model for this task.`);
|
||||
}
|
||||
const adapter = getAdapter(resolved.provider.kind);
|
||||
const purpose = opts.purpose ?? opts.task;
|
||||
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
const started = now();
|
||||
let streamed = false;
|
||||
try {
|
||||
for await (const event of adapter.stream(resolved.provider, req)) {
|
||||
if (event.type === 'done') {
|
||||
recordUsage(app, resolved, purpose, event.usage, started);
|
||||
const text = event.message.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
|
||||
return { message: event.message, text, stopReason: event.stopReason, usage: event.usage, model: resolved.model };
|
||||
}
|
||||
streamed = true;
|
||||
opts.onEvent?.(event);
|
||||
}
|
||||
throw new AIError('invalid_response', `${resolved.provider.name} ended the response unexpectedly.`, { retryable: true });
|
||||
} catch (raw) {
|
||||
const err = raw instanceof AIError ? raw : new AIError('server', 'The AI request failed unexpectedly.', { cause: raw });
|
||||
if (!(raw instanceof AIError)) app.log('error', 'Unexpected AI adapter error', raw);
|
||||
recordUsage(app, resolved, purpose, null, started, `${err.code}: ${err.message}`);
|
||||
if (!err.retryable || streamed || attempt >= 2 || opts.signal?.aborted) throw err;
|
||||
await sleep(1500 * (attempt + 1) ** 2, opts.signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Finds the first JSON value in model text (handles code fences and stray prose). */
|
||||
export function parseLooseJson(text: string): unknown {
|
||||
let t = text.trim();
|
||||
const fence = /```(?:json)?\s*\n?([\s\S]*?)```/i.exec(t);
|
||||
if (fence) t = fence[1].trim();
|
||||
const start = t.search(/[[{]/);
|
||||
if (start < 0) throw new AIError('invalid_response', 'The model did not return structured data.');
|
||||
t = t.slice(start);
|
||||
try {
|
||||
return JSON.parse(t);
|
||||
} catch {
|
||||
try {
|
||||
return JSON.parse(jsonrepair(t));
|
||||
} catch {
|
||||
throw new AIError('invalid_response', 'The model returned data that could not be read.', { retryable: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
|
||||
/** Name + description of the "function" whose arguments are the result. */
|
||||
toolName: string;
|
||||
toolDescription: string;
|
||||
schema: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured output that works on every provider: the schema is offered as a single tool the
|
||||
* model must call. Models without tool support (many local ones) get the schema in the prompt
|
||||
* and their text is parsed leniently instead.
|
||||
*/
|
||||
export async function generateStructured(app: AppContext, opts: StructuredOptions): Promise<{ data: unknown; usage: Usage; model: ModelDto; truncated: boolean }> {
|
||||
const resolved = resolveModel(app, opts.task, opts.modelId);
|
||||
const tool: ToolDef = { name: opts.toolName, description: opts.toolDescription, parameters: opts.schema };
|
||||
const promptJson = async () => {
|
||||
const system = `${opts.system ?? ''}\n\nRespond with ONLY a JSON object matching this JSON Schema. No prose, no Markdown fences.\n${JSON.stringify(opts.schema)}`.trim();
|
||||
const res = await runChat(app, { ...opts, system, jsonMode: true });
|
||||
return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
||||
};
|
||||
if (!resolved.model.capabilities.tools) return promptJson();
|
||||
try {
|
||||
const res = await runChat(app, { ...opts, tools: [tool], toolChoice: { name: opts.toolName } });
|
||||
const call = res.message.content.find((p) => p.type === 'tool_call');
|
||||
if (call && call.type === 'tool_call') return { data: call.input, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
||||
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
|
||||
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
|
||||
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
|
||||
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------ reporting ------------------------------ */
|
||||
|
||||
export function usageSummary(db: DB, days: number) {
|
||||
const since = now() - days * 86_400_000;
|
||||
const byModel = db.all<Record<string, any>>(
|
||||
`SELECT provider_name AS provider, model, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost,
|
||||
SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN cost IS NULL AND ok = 1 THEN 1 ELSE 0 END) AS unpriced
|
||||
FROM ai_usage WHERE at >= ? GROUP BY provider_name, model ORDER BY cost DESC, calls DESC`,
|
||||
since,
|
||||
);
|
||||
const byTask = db.all<Record<string, any>>(
|
||||
`SELECT task, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost FROM ai_usage WHERE at >= ? AND ok = 1 GROUP BY task ORDER BY cost DESC`,
|
||||
since,
|
||||
);
|
||||
const total = db.get<Record<string, any>>(
|
||||
`SELECT COUNT(*) AS calls, COALESCE(SUM(input_tokens), 0) AS input, COALESCE(SUM(output_tokens), 0) AS output, COALESCE(SUM(cost), 0) AS cost FROM ai_usage WHERE at >= ?`,
|
||||
since,
|
||||
)!;
|
||||
const recentErrors = db.all<Record<string, any>>(`SELECT at, provider_name AS provider, model, task, error FROM ai_usage WHERE ok = 0 AND at >= ? ORDER BY at DESC LIMIT 10`, since);
|
||||
return { days, total, byModel, byTask, recentErrors };
|
||||
}
|
||||
|
||||
/** True when at least one usable model exists; lets the UI show setup prompts instead of errors. */
|
||||
export function aiStatus(app: AppContext) {
|
||||
const models = listModels(app.db).filter((m) => m.available);
|
||||
const tasks = getTaskSettings(app.db);
|
||||
const resolved: Record<string, { modelId: string; label: string; provider: string } | null> = {};
|
||||
for (const t of TASKS) {
|
||||
try {
|
||||
const r = resolveModel(app, t);
|
||||
resolved[t] = { modelId: r.model.id, label: r.model.label, provider: r.provider.name };
|
||||
} catch {
|
||||
resolved[t] = null;
|
||||
}
|
||||
}
|
||||
return { ready: models.length > 0, modelCount: models.length, tasks, resolved };
|
||||
}
|
||||
180
server/ai/types.ts
Normal file
180
server/ai/types.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
/**
|
||||
* Provider-neutral AI contract.
|
||||
*
|
||||
* Every provider adapter (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama)
|
||||
* translates between this shape and the provider's wire format. Nothing outside
|
||||
* `server/ai/providers/` may depend on a provider-specific request or response.
|
||||
*/
|
||||
|
||||
export type ProviderKind = 'openai' | 'anthropic' | 'gemini' | 'openai_compatible' | 'ollama';
|
||||
|
||||
export const PROVIDER_KINDS: ProviderKind[] = ['openai', 'anthropic', 'gemini', 'openai_compatible', 'ollama'];
|
||||
|
||||
/** Decrypted provider connection, only ever held server-side. */
|
||||
export interface ProviderConfig {
|
||||
id: string;
|
||||
kind: ProviderKind;
|
||||
name: string;
|
||||
/** Null means "use the provider's official default endpoint". */
|
||||
baseUrl: string | null;
|
||||
apiKey: string | null;
|
||||
/** Extra HTTP headers (e.g. OpenRouter attribution, org ids). */
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
export type ContentPart =
|
||||
| { type: 'text'; text: string }
|
||||
/** Base64 image data (no data: prefix). */
|
||||
| { type: 'image'; mime: string; data: string }
|
||||
/** Base64 file, used for native PDF input where a provider supports it. */
|
||||
| { type: 'file'; mime: string; data: string; filename?: string }
|
||||
| { type: 'tool_call'; id: string; name: string; input: Record<string, unknown> }
|
||||
| { type: 'tool_result'; toolCallId: string; name: string; output: string; isError?: boolean };
|
||||
|
||||
export interface ChatMessage {
|
||||
/** Tool results travel as `tool_result` parts inside a `user` message. */
|
||||
role: 'user' | 'assistant';
|
||||
content: ContentPart[];
|
||||
/**
|
||||
* Opaque provider state for an assistant turn (thinking blocks + signatures,
|
||||
* Gemini thought signatures, OpenAI reasoning items...). An adapter writes it
|
||||
* on the `done` event and, when the next request goes to the same provider
|
||||
* kind, replays it verbatim instead of rebuilding the turn from `content`.
|
||||
*/
|
||||
providerData?: { kind: ProviderKind; data: unknown };
|
||||
}
|
||||
|
||||
export interface ToolDef {
|
||||
name: string;
|
||||
description: string;
|
||||
/** JSON Schema (object). Adapters down-convert for providers with a restricted dialect. */
|
||||
parameters: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type ReasoningEffort = 'none' | 'low' | 'medium' | 'high';
|
||||
|
||||
export interface ModelCapabilities {
|
||||
tools: boolean;
|
||||
vision: boolean;
|
||||
/** Accepts PDF files directly as input. */
|
||||
pdf: boolean;
|
||||
/** Has a configurable reasoning / thinking effort. */
|
||||
reasoning: boolean;
|
||||
/** Accepts a custom temperature (some reasoning models reject it). */
|
||||
temperature: boolean;
|
||||
}
|
||||
|
||||
export interface ChatRequest {
|
||||
model: string;
|
||||
system?: string;
|
||||
messages: ChatMessage[];
|
||||
tools?: ToolDef[];
|
||||
toolChoice?: 'auto' | 'none' | 'required' | { name: string };
|
||||
temperature?: number;
|
||||
maxOutputTokens?: number;
|
||||
reasoningEffort?: ReasoningEffort;
|
||||
/** Ask for a JSON object as the text output when the provider has a JSON mode. */
|
||||
jsonMode?: boolean;
|
||||
/** Known capabilities of the model, when the caller has them. Adapters must not send unsupported params. */
|
||||
capabilities?: Partial<ModelCapabilities>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface Usage {
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheReadTokens?: number;
|
||||
cacheWriteTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}
|
||||
|
||||
export type StopReason = 'end' | 'tool_use' | 'max_tokens' | 'refusal' | 'other';
|
||||
|
||||
export type StreamEvent =
|
||||
| { type: 'text'; delta: string }
|
||||
| { type: 'reasoning'; delta: string }
|
||||
/** Emitted once per tool call, after its arguments are complete and parsed. */
|
||||
| { type: 'tool_call'; call: { id: string; name: string; input: Record<string, unknown> } }
|
||||
/** Always the final event of a successful stream. */
|
||||
| { type: 'done'; message: ChatMessage; stopReason: StopReason; usage: Usage };
|
||||
|
||||
export interface DiscoveredModel {
|
||||
/** Identifier sent to the API. */
|
||||
id: string;
|
||||
label?: string;
|
||||
contextWindow?: number;
|
||||
maxOutput?: number;
|
||||
capabilities?: Partial<ModelCapabilities>;
|
||||
/** USD per million tokens, when known. */
|
||||
priceIn?: number;
|
||||
priceOut?: number;
|
||||
}
|
||||
|
||||
export interface TestResult {
|
||||
ok: boolean;
|
||||
message: string;
|
||||
latencyMs: number;
|
||||
/** Number of models visible to the key, when discovery works. */
|
||||
modelCount?: number;
|
||||
}
|
||||
|
||||
export interface ProviderAdapter {
|
||||
kind: ProviderKind;
|
||||
/** Human label and defaults used by the settings UI. */
|
||||
meta: {
|
||||
label: string;
|
||||
defaultBaseUrl: string | null;
|
||||
requiresApiKey: boolean;
|
||||
requiresBaseUrl: boolean;
|
||||
supportsDiscovery: boolean;
|
||||
docsUrl: string;
|
||||
keyHint: string;
|
||||
};
|
||||
/** Streams one model turn. Throws AIError; never yields after `done`. */
|
||||
stream(cfg: ProviderConfig, req: ChatRequest): AsyncIterable<StreamEvent>;
|
||||
/** Lists chat-capable models. Throws AIError('unsupported') when a provider has no listing API. */
|
||||
listModels(cfg: ProviderConfig, signal?: AbortSignal): Promise<DiscoveredModel[]>;
|
||||
/** Cheap credential / connectivity check. Never throws. */
|
||||
test(cfg: ProviderConfig, model?: string): Promise<TestResult>;
|
||||
}
|
||||
|
||||
export type AIErrorCode =
|
||||
| 'auth'
|
||||
| 'rate_limit'
|
||||
| 'quota'
|
||||
| 'context_length'
|
||||
| 'not_found'
|
||||
| 'bad_request'
|
||||
| 'content_filter'
|
||||
| 'server'
|
||||
| 'network'
|
||||
| 'timeout'
|
||||
| 'aborted'
|
||||
| 'unsupported'
|
||||
| 'invalid_response'
|
||||
| 'not_configured';
|
||||
|
||||
export class AIError extends Error {
|
||||
code: AIErrorCode;
|
||||
status?: number;
|
||||
retryable: boolean;
|
||||
/** Raw (already secret-scrubbed) provider message, for the details disclosure in the UI. */
|
||||
providerMessage?: string;
|
||||
|
||||
constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; cause?: unknown } = {}) {
|
||||
super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
|
||||
this.name = 'AIError';
|
||||
this.code = code;
|
||||
this.status = opts.status;
|
||||
this.retryable = opts.retryable ?? (code === 'rate_limit' || code === 'server' || code === 'network' || code === 'timeout');
|
||||
this.providerMessage = opts.providerMessage;
|
||||
}
|
||||
}
|
||||
|
||||
export const DEFAULT_CAPABILITIES: ModelCapabilities = {
|
||||
tools: true,
|
||||
vision: false,
|
||||
pdf: false,
|
||||
reasoning: false,
|
||||
temperature: true,
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue