Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

1000
server/ai/assistant.ts Normal file

File diff suppressed because it is too large Load diff

875
server/ai/generate.ts Normal file
View file

@ -0,0 +1,875 @@
/**
* Content generation: quizzes, flashcards and study guides from documents, pasted text,
* existing decks/quizzes, weak areas, or a plain topic.
*
* Every job writes into a draft row. Nothing reaches the library until the user has
* reviewed the draft and saved it. Long jobs are split into batches, each with its own
* slice of source material, so output limits and context windows are never a problem.
*/
import { z } from 'zod';
import {
validateDraftCard,
validateDraftQuestion,
type ContentDraft,
type DeckDraft,
type DraftCard,
type DraftQuestion,
type NoteDraft,
type QuizDraft,
} from '../../shared/content.ts';
import { draftsFromValue } from '../../shared/importer.ts';
import type { DB } from '../db.ts';
import { badRequest, notFound, type AppContext } from '../http.ts';
import { missedQuestionIds } from '../services/attempts.ts';
import {
createDeck,
createNote,
createQuiz,
draftCardToInput,
draftQuestionToInput,
getClass,
getDeck,
getQuiz,
insertCards,
insertQuestions,
listCards,
listQuestions,
updateQuestion,
} from '../services/content.ts';
import { chunkById, formatExcerpts, searchChunks, selectContext, sourceLabel, type ChunkHit } from '../services/documents.ts';
import { weakTopics } from '../services/stats.ts';
import { newId, now, parseJson } from '../util.ts';
import { generateStructured, resolveModel, runChat } from './service.ts';
import { AIError } from './types.ts';
/* ------------------------------ drafts ------------------------------ */
interface DraftRow {
id: string;
kind: 'quiz' | 'deck' | 'note';
class_id: string | null;
folder_id: string | null;
title: string;
payload: string;
meta: string;
status: 'generating' | 'ready' | 'failed' | 'saved' | 'discarded';
progress: number;
progress_label: string;
error: string;
saved_kind: string | null;
saved_id: string | null;
conversation_id: string | null;
created_at: number;
updated_at: number;
}
export interface DraftMeta {
origin?: 'generate' | 'assistant' | 'convert' | 'improve' | 'import';
model?: string;
request?: Record<string, unknown>;
sources?: { documentId: string; title: string }[];
/** For "improve" drafts: the stored question each draft question replaces (same order). */
replaceQuestionIds?: string[];
/** Suggested target when the draft is meant to extend an existing item. */
targetKind?: 'quiz' | 'deck';
targetId?: string;
notes?: string[];
}
function emptyPayload(kind: DraftRow['kind'], title = ''): ContentDraft {
if (kind === 'quiz') return { kind: 'quiz', title, description: '', questions: [] };
if (kind === 'deck') return { kind: 'deck', title, description: '', cards: [] };
return { kind: 'note', title, content: '' };
}
export function mapDraft(row: DraftRow) {
const payload = parseJson<ContentDraft>(row.payload, emptyPayload(row.kind, row.title));
return {
id: row.id,
kind: row.kind,
classId: row.class_id,
folderId: row.folder_id,
title: row.title,
payload,
meta: parseJson<DraftMeta>(row.meta, {}),
status: row.status,
progress: row.progress,
progressLabel: row.progress_label,
error: row.error,
savedKind: row.saved_kind,
savedId: row.saved_id,
conversationId: row.conversation_id,
itemCount: payload.kind === 'quiz' ? payload.questions.length : payload.kind === 'deck' ? payload.cards.length : payload.content.length,
createdAt: row.created_at,
updatedAt: row.updated_at,
};
}
export type DraftDto = ReturnType<typeof mapDraft>;
export function getDraft(db: DB, id: string): DraftDto {
const row = db.get<DraftRow>('SELECT * FROM drafts WHERE id = ?', id);
if (!row) throw notFound('Draft');
return mapDraft(row);
}
export function createDraft(
db: DB,
input: { kind: DraftRow['kind']; classId?: string | null; folderId?: string | null; title: string; payload?: ContentDraft; meta?: DraftMeta; status?: DraftRow['status']; conversationId?: string | null },
): string {
const id = newId();
const at = now();
db.run(
`INSERT INTO drafts (id, kind, class_id, folder_id, title, payload, meta, status, conversation_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
id,
input.kind,
input.classId ?? null,
input.folderId ?? null,
input.title.slice(0, 200),
JSON.stringify(input.payload ?? emptyPayload(input.kind, input.title)),
JSON.stringify(input.meta ?? {}),
input.status ?? 'ready',
input.conversationId ?? null,
at,
at,
);
return id;
}
export function listDrafts(db: DB) {
return db
.all<DraftRow>(`SELECT * FROM drafts WHERE status IN ('generating', 'ready', 'failed') ORDER BY updated_at DESC LIMIT 50`)
.map(mapDraft)
.map(({ payload: _payload, ...rest }) => rest);
}
export function updateDraftPayload(db: DB, id: string, payload: ContentDraft, extra: { classId?: string | null; folderId?: string | null } = {}): DraftDto {
const draft = getDraft(db, id);
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
if (payload.kind !== draft.kind) throw badRequest('Draft type cannot change');
db.run(
'UPDATE drafts SET payload = ?, title = ?, class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?',
JSON.stringify(payload),
(payload.title ?? '').slice(0, 200),
extra.classId === undefined ? draft.classId : extra.classId,
extra.folderId === undefined ? draft.folderId : extra.folderId,
now(),
id,
);
return getDraft(db, id);
}
export const draftSaveSchema = z.object({
classId: z.string().optional(),
folderId: z.string().nullish(),
/** new: create an item. append: add to an existing quiz/deck. replace: apply improved questions in place. */
mode: z.enum(['new', 'append', 'replace']).default('new'),
targetId: z.string().optional(),
});
/** Turns a reviewed draft into real content. Invalid items are rejected, never silently dropped. */
export function saveDraft(app: AppContext, id: string, input: z.input<typeof draftSaveSchema>): { kind: string; id: string; count: number } {
const { db } = app;
const opts = draftSaveSchema.parse(input);
const draft = getDraft(db, id);
if (draft.status === 'generating') throw badRequest('This draft is still being generated');
if (draft.status === 'saved' && draft.savedId) return { kind: draft.savedKind ?? draft.kind, id: draft.savedId, count: 0 };
const payload = draft.payload;
const classId = opts.classId ?? draft.classId;
let result: { kind: string; id: string; count: number };
if (payload.kind === 'quiz') {
const bad = payload.questions.flatMap((q, i) => validateDraftQuestion(q, i)).filter((x) => x.level === 'error');
if (bad.length > 0) throw badRequest(`Question ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
if (payload.questions.length === 0) throw badRequest('There are no questions to save');
const inputs = payload.questions.map(draftQuestionToInput);
if (opts.mode === 'replace') {
const ids = draft.meta.replaceQuestionIds ?? [];
const quizId = opts.targetId ?? draft.meta.targetId;
if (!quizId) throw badRequest('No quiz to update');
getQuiz(db, quizId);
db.tx(() => {
inputs.forEach((q, i) => {
const target = ids[i];
if (target && db.get('SELECT id FROM questions WHERE id = ? AND quiz_id = ?', target, quizId)) updateQuestion(db, target, q);
else insertQuestions(db, quizId, [q]);
});
});
result = { kind: 'quiz', id: quizId, count: inputs.length };
} else if (opts.mode === 'append') {
const quizId = opts.targetId ?? draft.meta.targetId;
if (!quizId) throw badRequest('Choose a quiz to add these questions to');
getQuiz(db, quizId);
insertQuestions(db, quizId, inputs);
result = { kind: 'quiz', id: quizId, count: inputs.length };
} else {
if (!classId) throw badRequest('Choose a class to save this quiz in');
const quiz = createQuiz(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled quiz', description: payload.description, questions: inputs });
result = { kind: 'quiz', id: quiz.id, count: inputs.length };
}
} else if (payload.kind === 'deck') {
const bad = payload.cards.flatMap((c, i) => validateDraftCard(c, i)).filter((x) => x.level === 'error');
if (bad.length > 0) throw badRequest(`Card ${(bad[0].index ?? 0) + 1}: ${bad[0].message}`, bad);
if (payload.cards.length === 0) throw badRequest('There are no cards to save');
const inputs = payload.cards.map(draftCardToInput);
if (opts.mode === 'append') {
const deckId = opts.targetId ?? draft.meta.targetId;
if (!deckId) throw badRequest('Choose a deck to add these cards to');
getDeck(db, deckId);
insertCards(db, deckId, inputs);
result = { kind: 'deck', id: deckId, count: inputs.length };
} else {
if (!classId) throw badRequest('Choose a class to save this deck in');
const deck = createDeck(db, { classId, folderId: opts.folderId ?? draft.folderId, title: payload.title.trim() || 'Untitled deck', description: payload.description, cards: inputs });
result = { kind: 'deck', id: deck.id, count: inputs.length };
}
} else {
if (!payload.content.trim()) throw badRequest('The note is empty');
if (!classId) throw badRequest('Choose a class to save this note in');
const note = createNote(db, {
classId,
folderId: opts.folderId ?? draft.folderId,
title: payload.title.trim() || 'Untitled note',
content: payload.content,
kind: 'guide',
source: { model: draft.meta.model, sources: draft.meta.sources },
});
result = { kind: 'note', id: note.id, count: 1 };
}
db.run(`UPDATE drafts SET status = 'saved', saved_kind = ?, saved_id = ?, updated_at = ? WHERE id = ?`, result.kind, result.id, now(), id);
return result;
}
export function discardDraft(db: DB, id: string): void {
cancelGeneration(id);
const { changes } = db.run(`UPDATE drafts SET status = 'discarded', updated_at = ? WHERE id = ?`, now(), id);
if (!changes) throw notFound('Draft');
}
/** Removes finished drafts after a while and marks jobs interrupted by a restart as failed. */
export function tidyDrafts(db: DB): void {
db.run(`UPDATE drafts SET status = 'failed', error = 'Generation was interrupted by a restart. Start it again.' WHERE status = 'generating'`);
db.run(`DELETE FROM drafts WHERE status IN ('saved', 'discarded') AND updated_at < ?`, now() - 14 * 86_400_000);
db.run(`DELETE FROM drafts WHERE status IN ('ready', 'failed') AND updated_at < ?`, now() - 60 * 86_400_000);
}
/* ------------------------------ request ------------------------------ */
export const generateRequestSchema = z.object({
kind: z.enum(['quiz', 'deck', 'note']),
classId: z.string().nullish(),
folderId: z.string().nullish(),
title: z.string().trim().max(200).optional(),
source: z
.object({
documentIds: z.array(z.string()).max(50).optional(),
/** Use every processed document in the class. */
allClassDocuments: z.boolean().optional(),
text: z.string().max(400_000).optional(),
quizId: z.string().optional(),
deckId: z.string().optional(),
noteId: z.string().optional(),
/** Build from the questions the user keeps missing. */
weakAreas: z.boolean().optional(),
})
.default({}),
/** Topic or focus, e.g. "pediatric respiratory disorders". */
focus: z.string().trim().max(600).optional(),
instructions: z.string().trim().max(4000).optional(),
count: z.number().int().min(1).max(200).optional(),
style: z.enum(['nclex', 'standard', 'recall']).optional(),
difficulty: z.enum(['easy', 'medium', 'hard', 'mixed']).optional(),
/** Share of select-all-that-apply questions, 0..1. */
sataShare: z.number().min(0).max(1).optional(),
cardStyle: z.enum(['concise', 'detailed', 'medication', 'definition']).optional(),
noteStyle: z.enum(['study_guide', 'summary', 'outline', 'cheat_sheet']).optional(),
modelId: z.string().nullish(),
});
export type GenerateRequest = z.infer<typeof generateRequestSchema>;
/* ------------------------------ prompts ------------------------------ */
const QUESTION_SCHEMA = {
type: 'object',
properties: {
questions: {
type: 'array',
items: {
type: 'object',
properties: {
type: { type: 'string', enum: ['single', 'multi'], description: 'single = one correct answer; multi = select all that apply' },
stem: { type: 'string', description: 'The question. Markdown allowed.' },
options: {
type: 'array',
items: {
type: 'object',
properties: {
text: { type: 'string' },
correct: { type: 'boolean' },
why: { type: 'string', description: 'One sentence on why this choice is right or wrong.' },
},
required: ['text', 'correct', 'why'],
},
},
rationale: { type: 'string', description: 'Teaching explanation of the correct answer. Markdown allowed.' },
topic: { type: 'string', description: 'Short topic label, reused across related questions (2-4 words).' },
difficulty: { type: 'string', enum: ['easy', 'medium', 'hard'] },
source: { type: 'string', description: 'Id of the excerpt this question is based on, e.g. "c42". Empty if none.' },
},
required: ['type', 'stem', 'options', 'rationale', 'topic', 'difficulty'],
},
},
},
required: ['questions'],
};
const CARD_SCHEMA = {
type: 'object',
properties: {
cards: {
type: 'array',
items: {
type: 'object',
properties: {
front: { type: 'string', description: 'Prompt side: one specific question or term.' },
back: { type: 'string', description: 'Answer side. Markdown allowed.' },
hint: { type: 'string', description: 'Optional memory hook or mnemonic. Empty string if none.' },
tags: { type: 'array', items: { type: 'string' }, description: '1-2 short topic tags.' },
},
required: ['front', 'back'],
},
},
},
required: ['cards'],
};
const NCLEX_RULES = `Write NCLEX-style (Next Generation NCLEX) items:
- Test clinical judgment at the application and analysis level: recognise cues, analyse, prioritise, choose and evaluate actions. Avoid pure recall.
- Open with a brief, realistic client scenario containing only relevant data (age, diagnosis, key findings, vital signs or lab values with units).
- Ask one clear question. Use priority wording where it fits (first, best, most important, requires immediate follow-up) and build it on a real framework: ABCs, Maslow, the nursing process, acute before chronic, unstable before stable, safety, least invasive first.
- Cover the client-needs areas where the material allows: safe and effective care, pharmacology, physiological adaptation, reduction of risk, health promotion, psychosocial integrity, delegation and assignment.
- Every distractor must be plausible to a student with partial knowledge and wrong for a specific reason. Keep choices parallel in grammar and similar in length. Never use "all of the above" or "none of the above". Do not give the answer away through wording or length.
- Multiple-choice items have exactly 4 choices with exactly one best answer. Select-all-that-apply items have 5 or 6 choices with 2 to 5 correct, and the stem ends with "Select all that apply."
- Use generic drug names. Use current, standard practice.
- The rationale teaches: state why the correct answer is correct, name the principle behind it, and give each choice its own "why".`;
const STANDARD_RULES = `Write clear exam-quality questions:
- Each question tests one idea and has one defensible answer set.
- Mix understanding and application; avoid trivia and trick wording.
- Distractors are plausible and parallel. Never use "all of the above" or "none of the above".
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
- The rationale explains the correct answer, and each choice gets its own "why".`;
const RECALL_RULES = `Write straightforward recall questions that check key facts, definitions, normal values and classifications.
- Multiple-choice items have 4 choices with exactly one correct. Select-all items have 4 to 6 choices with at least 2 correct.
- Keep stems short and unambiguous. Each choice gets a brief "why".`;
const CARD_RULES: Record<string, string> = {
concise: 'Each card tests exactly one fact. Fronts are specific questions or terms; backs are short (a phrase or one or two lines).',
detailed: 'Each card covers one concept. Backs may use a short bullet list (up to 5 bullets) for the key points.',
medication:
'Make medication cards. Front: the generic drug name (with a common brand name in parentheses when well known). Back: a Markdown list with **Class**, **Action**, **Uses**, **Adverse effects** (the most important ones), **Nursing considerations** and **Client teaching**. Only include what the material supports.',
definition: 'Make term and definition cards. Front: the term. Back: a precise definition in one or two sentences, plus an example when useful.',
};
const NOTE_RULES: Record<string, string> = {
study_guide:
'Write a study guide in Markdown: a one-paragraph overview, then sections with headings. Under each heading give the key concepts as tight bullet points, important values and medications in tables where that helps, common pitfalls, and finish with a "Check yourself" list of 5-10 self-test questions (answers at the very end).',
summary: 'Write a faithful summary in Markdown: a short overview paragraph, then the main points grouped under headings. Keep it compact and do not add outside information.',
outline: 'Write a hierarchical outline in Markdown using nested bullet points. Capture the structure and every key point of the material without full sentences where a phrase will do.',
cheat_sheet: 'Write a one-page cheat sheet in Markdown: dense, scannable, organised under short headings, using tables for values, drug classes and comparisons, and bold for must-know facts.',
};
function baseSystem(req: GenerateRequest, hasSources: boolean): string {
const lines = [
'You are an expert educator writing study material for a nursing student. Accuracy is critical: this material is used to prepare for licensing exams.',
hasSources
? 'Base everything on the provided source material. Do not introduce facts that contradict it. You may rely on standard, well-established knowledge only to write plausible distractors and clear explanations.'
: 'No source material was provided, so rely on current, well-established knowledge. If something is uncertain or varies by institution, prefer the most widely taught standard.',
'Write in clear, natural English. Use Markdown inside text fields where it aids clarity (bold key terms, short lists, tables). Do not wrap output in code fences.',
];
if (req.instructions) lines.push(`Additional instructions from the student (follow them unless they conflict with accuracy):\n${req.instructions}`);
return lines.join('\n\n');
}
/* ------------------------------ source gathering ------------------------------ */
interface Sources {
/** Batches of excerpts (documents). */
groups: ChunkHit[][];
/** Free text used as-is (pasted notes, an existing deck or quiz, missed questions). */
inline: string;
docs: { documentId: string; title: string }[];
refs: Map<string, string>;
totalTokens: number;
}
function gatherSources(app: AppContext, req: GenerateRequest, budgetTokens: number, groups: number): Sources {
const { db } = app;
const src = req.source;
const out: Sources = { groups: [], inline: '', docs: [], refs: new Map(), totalTokens: 0 };
const inline: string[] = [];
let documentIds = src.documentIds ?? [];
if (src.allClassDocuments && req.classId) {
documentIds = db.all<{ id: string }>(`SELECT id FROM documents WHERE class_id = ? AND status = 'ready' AND chunk_count > 0 ORDER BY sort_order, created_at`, req.classId).map((d) => d.id);
}
if (documentIds.length > 0) {
const rows = db.all<{ id: string; title: string; status: string; chunk_count: number }>(
`SELECT id, title, status, chunk_count FROM documents WHERE id IN (${documentIds.map(() => '?').join(',')})`,
...documentIds,
);
const usable = rows.filter((r) => r.status === 'ready' && r.chunk_count > 0);
if (usable.length === 0) {
throw badRequest(rows.some((r) => r.status !== 'ready') ? 'Those documents are still being processed. Try again in a moment.' : 'Those documents have no readable text yet.');
}
out.docs = usable.map((r) => ({ documentId: r.id, title: r.title }));
out.groups = selectContext(db, { documentIds: usable.map((r) => r.id) }, { focus: req.focus, budgetTokens, groups });
}
if (src.text?.trim()) inline.push(`Notes provided by the student:\n${src.text.trim()}`);
if (src.noteId) {
const note = db.get<{ title: string; content: string }>('SELECT title, content FROM notes WHERE id = ?', src.noteId);
if (note) inline.push(`Note "${note.title}":\n${note.content}`);
}
if (src.deckId) {
const deck = getDeck(db, src.deckId);
const cards = listCards(db, deck.id);
if (cards.length === 0) throw badRequest('That deck has no cards');
inline.push(`Flashcards from the deck "${deck.title}":\n${cards.map((c, i) => `${i + 1}. ${c.front}\n -> ${c.back}`).join('\n')}`);
}
if (src.quizId) {
const quiz = getQuiz(db, src.quizId);
const questions = listQuestions(db, quiz.id);
if (questions.length === 0) throw badRequest('That quiz has no questions');
inline.push(
`Questions from the quiz "${quiz.title}":\n${questions
.map((q, i) => `${i + 1}. ${q.stem}\n Correct: ${q.options.filter((o) => o.correct).map((o) => o.text).join('; ')}\n Rationale: ${q.rationale}`)
.join('\n')}`,
);
}
if (src.weakAreas) {
const ids = missedQuestionIds(db, { classId: req.classId ?? undefined }, 40);
const topics = weakTopics(db, { classId: req.classId ?? undefined }, 10);
if (ids.length === 0 && topics.length === 0) throw badRequest('There is not enough quiz history yet to find weak areas. Take a few quizzes first.');
const missed = ids
.map((id) => db.get<{ stem: string; topic: string; rationale: string }>('SELECT stem, topic, rationale FROM questions WHERE id = ?', id))
.filter((q): q is { stem: string; topic: string; rationale: string } => Boolean(q));
inline.push(
[
'The student keeps missing these concepts. Write NEW items that test the same underlying knowledge from different angles. Do not repeat or lightly reword the original questions.',
topics.length ? `Weakest topics: ${topics.map((t) => `${t.topic} (${Math.round(t.accuracy * 100)}% correct)`).join(', ')}` : '',
missed.length ? `Questions answered incorrectly:\n${missed.map((q, i) => `${i + 1}. [${q.topic || 'General'}] ${q.stem}\n Key point: ${q.rationale.slice(0, 400)}`).join('\n')}` : '',
]
.filter(Boolean)
.join('\n\n'),
);
// Pull supporting passages from the class documents for those topics.
if (req.classId && out.groups.length === 0) {
const query = [...topics.map((t) => t.topic), ...missed.slice(0, 10).map((q) => q.topic)].filter(Boolean).join(' ');
const hits = query ? searchChunks(db, query, { classId: req.classId }, 24) : [];
if (hits.length > 0) {
const perGroup = Math.max(1, Math.ceil(hits.length / groups));
for (let i = 0; i < hits.length; i += perGroup) out.groups.push(hits.slice(i, i + perGroup));
}
}
}
out.inline = inline.join('\n\n');
for (const g of out.groups) for (const c of g) out.refs.set(`c${c.id}`, sourceLabel(c));
out.totalTokens = out.groups.reduce((n, g) => n + g.reduce((m, c) => m + c.tokens, 0), 0) + Math.ceil(out.inline.length / 4);
return out;
}
function sourceBlock(sources: Sources, groupIndex: number): string {
const parts: string[] = [];
const group = sources.groups.length > 0 ? sources.groups[groupIndex % sources.groups.length] : [];
if (group.length > 0) parts.push(`SOURCE EXCERPTS (each starts with its id in brackets):\n\n${formatExcerpts(group)}`);
if (sources.inline) parts.push(sources.inline);
return parts.join('\n\n======\n\n');
}
/* ------------------------------ job control ------------------------------ */
const running = new Map<string, AbortController>();
export function cancelGeneration(draftId: string): void {
running.get(draftId)?.abort();
running.delete(draftId);
}
function setProgress(db: DB, id: string, progress: number, label: string, payload?: ContentDraft): void {
if (payload) {
db.run('UPDATE drafts SET progress = ?, progress_label = ?, payload = ?, updated_at = ? WHERE id = ?', progress, label, JSON.stringify(payload), now(), id);
} else {
db.run('UPDATE drafts SET progress = ?, progress_label = ?, updated_at = ? WHERE id = ?', progress, label, now(), id);
}
}
function defaultTitle(app: AppContext, req: GenerateRequest, sources: Sources): string {
if (req.title) return req.title;
const subject =
req.focus?.replace(/\s+/g, ' ').slice(0, 80) ||
(sources.docs.length === 1 ? sources.docs[0].title : '') ||
(req.source.deckId ? getDeck(app.db, req.source.deckId).title : '') ||
(req.source.quizId ? getQuiz(app.db, req.source.quizId).title : '') ||
(req.source.weakAreas ? 'Weak areas' : '') ||
(req.classId ? getClass(app.db, req.classId).name : 'Untitled');
const suffix = req.kind === 'quiz' ? (req.source.weakAreas ? 'practice' : req.style === 'nclex' ? 'NCLEX practice' : 'quiz') : req.kind === 'deck' ? 'flashcards' : NOTE_TITLES[req.noteStyle ?? 'study_guide'];
const cap = subject.charAt(0).toUpperCase() + subject.slice(1);
return `${cap} · ${suffix}`.slice(0, 200);
}
const NOTE_TITLES: Record<string, string> = { study_guide: 'study guide', summary: 'summary', outline: 'outline', cheat_sheet: 'cheat sheet' };
/**
* Starts a generation job in the background and returns the draft id immediately.
* The caller polls the draft (or awaits `done`) for progress.
*/
export function startGeneration(app: AppContext, input: z.input<typeof generateRequestSchema>, opts: { conversationId?: string | null; origin?: DraftMeta['origin'] } = {}): { draftId: string; done: Promise<DraftDto> } {
const req = generateRequestSchema.parse(input);
if (req.classId) getClass(app.db, req.classId);
// Fail fast (before creating a draft) when no model is configured.
const resolved = resolveModel(app, 'generate', req.modelId);
const draftId = createDraft(app.db, {
kind: req.kind,
classId: req.classId ?? null,
folderId: req.folderId ?? null,
title: req.title ?? '',
status: 'generating',
conversationId: opts.conversationId ?? null,
meta: { origin: opts.origin ?? 'generate', model: resolved.model.label, request: { ...req, source: { ...req.source, text: req.source.text ? `${req.source.text.slice(0, 200)}…` : undefined } } },
});
const controller = new AbortController();
running.set(draftId, controller);
const done = runGeneration(app, draftId, req, controller.signal)
.catch((err) => {
const message = err instanceof AIError ? err.message : err instanceof Error && 'status' in err ? err.message : 'Generation failed unexpectedly.';
if (!(err instanceof AIError) && !(err instanceof Error && 'status' in err)) app.log('error', 'Generation failed', err);
const current = app.db.get<{ status: string }>('SELECT status FROM drafts WHERE id = ?', draftId);
if (current?.status === 'generating') {
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ?`, controller.signal.aborted ? 'Generation was cancelled.' : message, now(), draftId);
}
})
.then(() => {
running.delete(draftId);
return getDraft(app.db, draftId);
});
return { draftId, done };
}
async function runGeneration(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal): Promise<void> {
const { db } = app;
const resolved = resolveModel(app, 'generate', req.modelId);
const window = resolved.model.contextWindow ?? 32_000;
const budgetTokens = Math.max(2_500, Math.min(Math.floor(window * 0.4), 28_000));
setProgress(db, draftId, 0.03, 'Reading your material');
if (req.kind === 'note') {
await generateNote(app, draftId, req, signal, budgetTokens, window);
return;
}
const isQuiz = req.kind === 'quiz';
const perBatch = isQuiz ? 8 : 25;
const defaultCount = isQuiz ? 15 : 30;
let count = req.count ?? defaultCount;
if ((req.source.deckId || req.source.quizId) && !req.count) {
// Conversions default to one item per source item.
count = req.source.deckId ? listCards(db, req.source.deckId).length : listQuestions(db, req.source.quizId!).length;
count = Math.min(Math.max(count, 1), 100);
}
const batches = Math.ceil(count / perBatch);
const sources = gatherSources(app, req, budgetTokens, batches);
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
if (!hasSources && !req.focus && !req.instructions) throw badRequest('Choose source material or describe a topic to generate from.');
const title = defaultTitle(app, req, sources);
const draft: QuizDraft | DeckDraft = isQuiz ? { kind: 'quiz', title, description: '', questions: [] } : { kind: 'deck', title, description: '', cards: [] };
const items = (): (DraftQuestion | DraftCard)[] => (draft.kind === 'quiz' ? draft.questions : draft.cards);
const key = (x: DraftQuestion | DraftCard) => ('stem' in x ? x.stem : x.front).toLowerCase().replace(/\W+/g, ' ').trim().slice(0, 120);
const seen = new Set<string>();
const notes: string[] = [];
const style = req.style ?? 'nclex';
const rules = isQuiz ? (style === 'nclex' ? NCLEX_RULES : style === 'recall' ? RECALL_RULES : STANDARD_RULES) : CARD_RULES[req.cardStyle ?? 'concise'];
const system = `${baseSystem(req, hasSources)}\n\n${rules}`;
const sata = req.sataShare ?? (style === 'nclex' ? 0.3 : 0.2);
const difficulty = req.difficulty ?? 'mixed';
const runBatch = async (index: number, want: number): Promise<number> => {
const written = items().map((x) => `- ${('stem' in x ? x.stem : x.front).replace(/\s+/g, ' ').slice(0, 140)}`);
const sataCount = Math.round(want * sata);
const ask = isQuiz
? [
`Write ${want} ${style === 'nclex' ? 'NCLEX-style ' : ''}questions${req.focus ? ` about: ${req.focus}` : ''}.`,
sata <= 0 ? 'Use only multiple-choice (single) questions.' : sata >= 1 ? 'Use only select-all-that-apply (multi) questions.' : `About ${sataCount} of them should be select-all-that-apply (multi); the rest multiple choice (single).`,
difficulty === 'mixed' ? 'Mix difficulty: mostly medium, with a few easy and a few hard.' : `Difficulty: ${difficulty}.`,
sources.groups.length > 0 ? 'Spread the questions across the excerpts instead of clustering on one. Set "source" to the id of the excerpt each question is based on.' : '',
req.source.deckId ? 'Turn the flashcards into questions: use each card\'s answer as the correct choice and write distractors that a student could confuse with it.' : '',
]
: [
`Write ${want} flashcards${req.focus ? ` about: ${req.focus}` : ''}.`,
sources.groups.length > 0 ? 'Cover the most testable facts across all of the excerpts.' : '',
req.source.quizId ? 'Turn the quiz questions into flashcards: the front asks for the key fact, the back states it with the essential reasoning.' : '',
];
if (written.length > 0) ask.push(`Already written (do not repeat or closely paraphrase these):\n${written.slice(-80).join('\n')}`);
const user = [sourceBlock(sources, index), ask.filter(Boolean).join('\n')].filter(Boolean).join('\n\n======\n\n');
const result = await generateStructured(app, {
task: 'generate',
purpose: isQuiz ? 'quiz generation' : 'flashcard generation',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: user }] }],
toolName: isQuiz ? 'submit_questions' : 'submit_flashcards',
toolDescription: isQuiz ? 'Submit the finished questions.' : 'Submit the finished flashcards.',
schema: isQuiz ? QUESTION_SCHEMA : CARD_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal,
});
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === req.kind) as QuizDraft | DeckDraft | undefined;
const incoming: (DraftQuestion | DraftCard)[] = parsed ? (parsed.kind === 'quiz' ? parsed.questions : parsed.cards) : [];
let added = 0;
for (const item of incoming) {
const k = key(item);
if (!k || seen.has(k)) continue;
const problems = ('stem' in item ? validateDraftQuestion(item, 0) : validateDraftCard(item, 0)).filter((p) => p.level === 'error');
if (problems.length > 0) continue; // malformed items are dropped here and replaced by a top-up batch
if ('stem' in item) {
const ref = (item.source ?? '').replace(/[[\]\s]/g, '');
item.source = sources.refs.get(ref) ?? undefined;
}
seen.add(k);
(items() as (DraftQuestion | DraftCard)[]).push(item);
added += 1;
if (items().length >= count) break;
}
if (result.truncated) notes.push('One batch hit the model output limit, so it produced fewer items than requested.');
return added;
};
let failures = 0;
for (let i = 0; i < batches && items().length < count; i += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
const want = Math.min(perBatch, count - items().length);
setProgress(db, draftId, 0.05 + (0.9 * items().length) / count, `Writing ${isQuiz ? 'questions' : 'cards'} ${items().length + 1}-${Math.min(count, items().length + want)} of ${count}`, draft);
try {
await runBatch(i, want);
} catch (err) {
// One bad batch should not sink a long job that already has good content.
failures += 1;
if (items().length === 0 || failures > 2 || (err instanceof AIError && !['invalid_response', 'server', 'rate_limit', 'timeout', 'network'].includes(err.code))) throw err;
notes.push(`A batch failed and was skipped: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
// Top up when validation or de-duplication left the draft short.
for (let extra = 0; extra < 2 && items().length < count && items().length > 0; extra += 1) {
if (signal.aborted) break;
const want = Math.min(perBatch, count - items().length);
setProgress(db, draftId, 0.92, `Writing the last ${want} ${isQuiz ? 'questions' : 'cards'}`, draft);
try {
if ((await runBatch(batches + extra, want)) === 0) break;
} catch {
break;
}
}
if (items().length === 0) throw new AIError('invalid_response', 'The model did not produce any usable items. Try again, or choose a different model for content generation.');
if (items().length < count) notes.push(`Generated ${items().length} of the ${count} requested.`);
db.run(
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
draft.title,
JSON.stringify(draft),
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs, notes: [...new Set(notes)] } satisfies DraftMeta),
now(),
draftId,
);
}
async function generateNote(app: AppContext, draftId: string, req: GenerateRequest, signal: AbortSignal, budgetTokens: number, window: number): Promise<void> {
const { db } = app;
const resolved = resolveModel(app, 'generate', req.modelId);
const style = req.noteStyle ?? 'study_guide';
// A guide reads the material in as few passes as possible; allow a bigger slice per pass than item generation.
const perPass = Math.max(budgetTokens, Math.min(Math.floor(window * 0.55), 90_000));
const sources = gatherSources(app, req, perPass, 12);
const hasSources = sources.groups.length > 0 || Boolean(sources.inline);
if (!hasSources && !req.focus) throw badRequest('Choose source material or describe a topic to write about.');
const title = defaultTitle(app, req, sources);
const system = `${baseSystem(req, hasSources)}\n\n${NOTE_RULES[style]}\n\n${
sources.groups.length > 0 ? 'Cite the excerpts you draw on by putting their id in brackets at the end of the relevant line or paragraph, e.g. [c42]. Use only ids that appear in the source.' : ''
}\nStart directly with the content (no preamble, no title line).`;
// Merge excerpt groups into as few passes as fit.
const passes: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const g of sources.groups) {
const t = g.reduce((n, c) => n + c.tokens, 0);
if (current.length > 0 && size + t > perPass) {
passes.push(current);
current = [];
size = 0;
}
current.push(...g);
size += t;
}
if (current.length > 0) passes.push(current);
const ask = `${req.focus ? `Focus on: ${req.focus}\n` : ''}Write the ${NOTE_TITLES[style]} now.`;
let content = '';
if (passes.length <= 1) {
setProgress(db, draftId, 0.2, 'Writing');
const body = [passes[0]?.length ? `SOURCE EXCERPTS:\n\n${formatExcerpts(passes[0])}` : '', sources.inline, ask].filter(Boolean).join('\n\n======\n\n');
const res = await runChat(app, { task: 'generate', purpose: 'study guide', modelId: req.modelId, system, messages: [{ role: 'user', content: [{ type: 'text', text: body }] }], maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000, signal });
content = res.text;
} else {
// Long material: condense each part, then write the final piece from the condensed notes.
const partials: string[] = [];
for (let i = 0; i < passes.length; i += 1) {
if (signal.aborted) throw new AIError('aborted', 'Generation was cancelled.');
setProgress(db, draftId, 0.05 + (0.7 * i) / passes.length, `Reading part ${i + 1} of ${passes.length}`);
const res = await runChat(app, {
task: 'generate',
purpose: 'study guide',
modelId: req.modelId,
system: `${baseSystem(req, true)}\n\nCondense this part of the material into detailed notes: every key concept, value, medication, cause, sign, intervention and warning, as Markdown bullet points under short headings. Keep the bracketed excerpt ids (e.g. [c42]) next to the points they support. Do not add outside information.`,
messages: [{ role: 'user', content: [{ type: 'text', text: `${req.focus ? `Focus on: ${req.focus}\n\n` : ''}SOURCE EXCERPTS (part ${i + 1} of ${passes.length}):\n\n${formatExcerpts(passes[i])}` }] }],
maxOutputTokens: 6_000,
signal,
});
partials.push(res.text.trim());
}
setProgress(db, draftId, 0.8, 'Writing');
const res = await runChat(app, {
task: 'generate',
purpose: 'study guide',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: `CONDENSED NOTES FROM THE FULL MATERIAL:\n\n${partials.join('\n\n---\n\n')}\n\n======\n\n${sources.inline ? `${sources.inline}\n\n======\n\n` : ''}${ask}` }] }],
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal,
});
content = res.text;
}
content = content.trim().replace(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i, '$1');
if (!content) throw new AIError('invalid_response', 'The model returned an empty document. Try again.');
// Turn [c42] markers into readable citations and append a source list.
const used = new Map<string, string>();
content = content.replace(/\[(c\d+(?:\s*,\s*c\d+)*)\]/g, (_m, list: string) => {
const labels = list
.split(',')
.map((r) => r.trim())
.map((r) => {
const label = sources.refs.get(r) ?? (() => {
const hit = chunkById(db, Number(r.slice(1)));
return hit ? sourceLabel(hit) : null;
})();
if (label) used.set(label, label);
return label;
})
.filter(Boolean);
return labels.length ? ` *(${[...new Set(labels)].join('; ')})*` : '';
});
const payload: NoteDraft = { kind: 'note', title, content };
db.run(
`UPDATE drafts SET status = 'ready', title = ?, payload = ?, meta = ?, progress = 1, progress_label = '', error = '', updated_at = ? WHERE id = ?`,
title,
JSON.stringify(payload),
JSON.stringify({ ...getDraft(db, draftId).meta, model: resolved.model.label, sources: sources.docs } satisfies DraftMeta),
now(),
draftId,
);
}
/* ------------------------------ improve existing questions ------------------------------ */
export const improveRequestSchema = z.object({
quizId: z.string(),
questionIds: z.array(z.string()).max(60).optional(),
instructions: z.string().trim().max(2000).optional(),
modelId: z.string().nullish(),
});
/** Rewrites existing questions (clearer stems, better distractors, fuller rationales) into a draft the user can apply. */
export function startImprove(app: AppContext, input: z.input<typeof improveRequestSchema>, opts: { conversationId?: string | null } = {}): { draftId: string; done: Promise<DraftDto> } {
const req = improveRequestSchema.parse(input);
const quiz = getQuiz(app.db, req.quizId);
let questions = listQuestions(app.db, quiz.id);
if (req.questionIds?.length) questions = questions.filter((q) => req.questionIds!.includes(q.id));
if (questions.length === 0) throw badRequest('No questions to improve');
questions = questions.slice(0, 60);
const resolved = resolveModel(app, 'generate', req.modelId);
const draftId = createDraft(app.db, {
kind: 'quiz',
classId: quiz.class_id,
title: quiz.title,
status: 'generating',
conversationId: opts.conversationId ?? null,
meta: { origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: [] },
});
const controller = new AbortController();
running.set(draftId, controller);
const done = (async () => {
const out: DraftQuestion[] = [];
const ids: string[] = [];
const system = `You are an expert nursing educator and item writer reviewing existing exam questions.
Improve each question while keeping what it tests and its correct answer(s) unless the keyed answer is factually wrong (if so, fix it and say so at the start of the rationale).
- Make the stem clear, complete and free of clues; fix grammar.
- Replace weak or implausible distractors; keep choices parallel and similar in length; remove "all/none of the above".
- Write a teaching rationale, and give every choice a one-sentence "why".
- Keep the same question type unless it is clearly wrong for the content. Set a short topic label and a difficulty.
Return the questions in the same order, one output question per input question.${req.instructions ? `\n\nAdditional instructions: ${req.instructions}` : ''}`;
for (let i = 0; i < questions.length; i += 8) {
if (controller.signal.aborted) throw new AIError('aborted', 'Cancelled.');
const batch = questions.slice(i, i + 8);
setProgress(app.db, draftId, 0.05 + (0.9 * i) / questions.length, `Reviewing questions ${i + 1}-${i + batch.length} of ${questions.length}`);
const body = batch
.map(
(q, n) =>
`Question ${n + 1} (${q.type})\nStem: ${q.stem}\nChoices:\n${q.options.map((o) => `- [${o.correct ? 'x' : ' '}] ${o.text}`).join('\n')}\nRationale: ${q.rationale || '(none)'}\nTopic: ${q.topic || '(none)'}`,
)
.join('\n\n');
const result = await generateStructured(app, {
task: 'generate',
purpose: 'improve questions',
modelId: req.modelId,
system,
messages: [{ role: 'user', content: [{ type: 'text', text: body }] }],
toolName: 'submit_questions',
toolDescription: 'Submit the improved questions in the original order.',
schema: QUESTION_SCHEMA,
maxOutputTokens: resolved.params.maxOutputTokens ?? 16_000,
signal: controller.signal,
});
const parsed = draftsFromValue(result.data).items.find((x) => x.kind === 'quiz') as QuizDraft | undefined;
const improved = parsed?.questions ?? [];
batch.forEach((orig, n) => {
const next = improved[n];
if (!next || validateDraftQuestion(next, 0).some((p) => p.level === 'error')) return;
next.source = undefined;
out.push(next);
ids.push(orig.id);
});
}
if (out.length === 0) throw new AIError('invalid_response', 'The model did not return usable revisions. Try again or use a different model.');
const payload: QuizDraft = { kind: 'quiz', title: quiz.title, description: quiz.description, questions: out };
app.db.run(
`UPDATE drafts SET status = 'ready', payload = ?, meta = ?, progress = 1, progress_label = '', updated_at = ? WHERE id = ?`,
JSON.stringify(payload),
JSON.stringify({ origin: 'improve', model: resolved.model.label, targetKind: 'quiz', targetId: quiz.id, replaceQuestionIds: ids } satisfies DraftMeta),
now(),
draftId,
);
})()
.catch((err) => {
const message = err instanceof AIError || (err instanceof Error && 'status' in err) ? (err as Error).message : 'Improving the questions failed unexpectedly.';
app.db.run(`UPDATE drafts SET status = 'failed', error = ?, updated_at = ? WHERE id = ? AND status = 'generating'`, message, now(), draftId);
})
.then(() => {
running.delete(draftId);
return getDraft(app.db, draftId);
});
return { draftId, done };
}

View file

@ -0,0 +1,581 @@
/**
* Anthropic — Messages API (`POST /v1/messages`, SSE).
*
* Verified against (2026-11):
* https://platform.claude.com/docs/en/api/messages
* https://platform.claude.com/docs/en/build-with-claude/streaming
* https://platform.claude.com/docs/en/build-with-claude/adaptive-thinking
* https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback
* https://platform.claude.com/docs/en/api/migration-2026-10-21
*
* Quirks handled here:
* - API version 2026-10-21 rejects temperature/top_p/top_k on EVERY model, so
* this adapter never sends a temperature (the catalog marks Claude models
* `temperature: false`).
* - `max_tokens` is required; thinking counts against it.
* - Thinking configuration differs per model generation (see `claudeProfile`).
* - Thinking blocks (with their signatures) must be replayed unchanged, so the
* raw content blocks of each turn are kept in `providerData`.
* - Newer models bind thinking blocks to the conversation prefix (system
* prompt, tool set, earlier messages). If the app changed any of those, the
* API answers 400 "Invalid `signature` in `thinking` block"; the documented
* recovery is to strip the thinking blocks and retry once.
* - Safety classifiers can end a turn with `stop_reason: "refusal"` (HTTP 200).
* For the models that support it we opt into server-side fallback, which
* re-runs a declined request on another Claude model inside the same call.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'Anthropic';
const DEFAULT_BASE_URL = 'https://api.anthropic.com';
const API_VERSION = '2026-10-21';
/** Beta that enables the `fallbacks` request parameter. */
const FALLBACK_BETA = 'server-side-fallback-2026-07-01';
/** Streaming has no HTTP-timeout concern, so give the model room (thinking counts against this). */
const DEFAULT_MAX_TOKENS = 64_000;
/* ------------------------------------------------------------------ */
/* Per-model behaviour */
/* ------------------------------------------------------------------ */
export interface ClaudeProfile {
/**
* always – thinking cannot be turned off (Fable/Mythos, Opus 5.5+)
* default_on – adaptive thinking runs unless disabled (Opus 5, Sonnet 5+, Haiku 5.5)
* opt_in – adaptive thinking must be requested (Opus 4.6–4.8, Sonnet 4.6)
* budget – legacy `budget_tokens` thinking (Haiku 4.5, Opus/Sonnet 4.5 and older)
* none – no thinking support (Claude 3 and earlier)
*/
thinking: 'always' | 'default_on' | 'opt_in' | 'budget' | 'none';
/** How thinking is switched off, when the model allows it. */
off: 'disabled' | 'between_tools' | null;
/** Accepts `output_config.effort`. */
effort: boolean;
/** Accepts `thinking.display` (and defaults to hiding the reasoning text). */
display: boolean;
/** Accepts `tool_choice` of type `any` / `tool`. */
forcedToolChoice: boolean;
/** Supports the server-side refusal `fallbacks` parameter. */
fallbacks: boolean;
}
/** Thinking / tool-choice rules by model, from the model migration guides. */
export function claudeProfile(model: string): ClaudeProfile {
const m = /claude-(fable|mythos|opus|sonnet|haiku)-(\d+)(?:-(\d{1,2}))?(?!\d)/.exec(model);
if (!m) {
// "claude-3-5-sonnet-…" and older put the version first and never had adaptive thinking.
if (/claude-(?:[123]\b|instant)/.test(model)) {
return { thinking: 'none', off: null, effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
// Unknown naming scheme: assume it behaves like the newest generation.
return { thinking: 'default_on', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: false };
}
const family = m[1];
const v = Number(m[2]) * 10 + Number(m[3] ?? 0); // 4.6 -> 46, 5 -> 50, 5.5 -> 55
if (family === 'fable' || family === 'mythos') {
return {
thinking: 'always',
off: null,
effort: true,
display: true,
forcedToolChoice: v <= 50,
fallbacks: family === 'fable' && (v === 51 || v === 52),
};
}
if (family === 'opus') {
if (v >= 55) return { thinking: 'always', off: null, effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: v === 50 };
if (v >= 47) return { thinking: 'opt_in', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: v === 45, display: false, forcedToolChoice: true, fallbacks: false };
}
if (family === 'sonnet') {
// Sonnet 5.5 rejects {type:"disabled"}; its lowest setting is {type:"between_tools"}.
if (v >= 55) return { thinking: 'default_on', off: 'between_tools', effort: true, display: true, forcedToolChoice: false, fallbacks: v === 55 };
if (v >= 50) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
if (v === 46) return { thinking: 'opt_in', off: 'disabled', effort: true, display: false, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
// haiku
if (v >= 55) return { thinking: 'default_on', off: 'disabled', effort: true, display: true, forcedToolChoice: true, fallbacks: false };
return { thinking: 'budget', off: 'disabled', effort: false, display: false, forcedToolChoice: true, fallbacks: false };
}
/** Thinking budgets for the legacy `budget_tokens` models. */
const LEGACY_BUDGET = { low: 2_048, medium: 8_192, high: 16_384 } as const;
/* ------------------------------------------------------------------ */
/* Request building */
/* ------------------------------------------------------------------ */
interface BuildOptions {
/** Official endpoint: enables features a proxy may not understand. */
official: boolean;
stripThinking: boolean;
useFallbacks: boolean;
eagerToolInput: boolean;
promptCache: boolean;
}
/** Tool-call ids must match ^[a-zA-Z0-9_-]+$; ids minted by other providers may not. */
function safeId(id: string): string {
const cleaned = id.replace(/[^a-zA-Z0-9_-]/g, '_');
return cleaned || 'tool_call';
}
function userBlocks(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const results: Json[] = [];
const rest: Json[] = [];
for (const p of parts) {
switch (p.type) {
case 'tool_result': {
// All results of one assistant turn go into ONE user message, before any other content.
const block: Json = { type: 'tool_result', tool_use_id: safeId(p.toolCallId), content: p.output || '(no output)' };
if (p.isError) block.is_error = true;
results.push(block);
break;
}
case 'text':
if (p.text) rest.push({ type: 'text', text: p.text });
break;
case 'image':
if (caps.vision === false) rest.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
else rest.push({ type: 'image', source: { type: 'base64', media_type: p.mime, data: p.data } });
break;
case 'file':
if (p.mime === 'application/pdf' && caps.pdf !== false) {
const doc: Json = { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: p.data } };
if (p.filename) doc.title = p.filename;
rest.push(doc);
} else if (p.mime.startsWith('text/')) {
const doc: Json = { type: 'document', source: { type: 'text', media_type: 'text/plain', data: Buffer.from(p.data, 'base64').toString('utf8') } };
if (p.filename) doc.title = p.filename;
rest.push(doc);
} else {
rest.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
break;
case 'tool_call':
break; // not valid in a user message
}
}
return [...results, ...rest];
}
function assistantBlocks(msg: ChatMessage, stripThinking: boolean): Json[] {
const pd = msg.providerData;
if (pd?.kind === 'anthropic' && isRecord(pd.data) && Array.isArray(pd.data.content)) {
// Same provider: replay the turn exactly as it was received, thinking blocks
// and signatures included.
const blocks = pd.data.content.filter(isRecord);
return stripThinking ? blocks.filter((b) => b.type !== 'thinking' && b.type !== 'redacted_thinking') : blocks;
}
// Another provider produced this turn: rebuild it from the neutral parts.
const blocks: Json[] = [];
for (const p of msg.content) {
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
else if (p.type === 'tool_call') blocks.push({ type: 'tool_use', id: safeId(p.id), name: p.name, input: p.input });
}
return blocks;
}
function toWireMessages(messages: ChatMessage[], caps: Partial<ModelCapabilities>, stripThinking: boolean): Json[] {
const out: Json[] = [];
for (const msg of messages) {
const content = msg.role === 'user' ? userBlocks(msg.content, caps) : assistantBlocks(msg, stripThinking);
if (content.length === 0) continue; // the API rejects empty turns
out.push({ role: msg.role, content });
}
if (out[0]?.role === 'assistant') out.unshift({ role: 'user', content: [{ type: 'text', text: '(continuing the conversation)' }] });
return out;
}
function buildBody(req: ChatRequest, profile: ClaudeProfile, opts: BuildOptions): Json {
const caps = req.capabilities ?? {};
const defaults = lookupModelDefaults('anthropic', req.model);
const ceiling = defaults.maxOutput ?? Number.POSITIVE_INFINITY;
const maxTokens = Math.max(1, Math.min(req.maxOutputTokens ?? DEFAULT_MAX_TOKENS, ceiling));
const systemNotes: string[] = [];
const body: Json = { model: req.model, max_tokens: maxTokens, stream: true };
// --- tools -------------------------------------------------------
const tools = caps.tools === false ? [] : (req.tools ?? []);
let forcing = false;
if (tools.length > 0) {
body.tools = tools.map((t) => {
const tool: Json = { name: t.name, description: t.description, input_schema: { type: 'object', ...t.parameters } };
// Without this the API buffers each tool-input parameter and the stream goes
// silent for the whole time a large input (a generated quiz, say) is written.
// The price is that the input is no longer validated server-side, so it is
// parsed strictly below and truncated calls are dropped.
if (opts.eagerToolInput) tool.eager_input_streaming = true;
return tool;
});
const choice = req.toolChoice;
if (choice === 'none') {
body.tool_choice = { type: 'none' };
} else if (choice === 'required' || isRecord(choice)) {
const name = isRecord(choice) ? str(choice.name) : '';
if (profile.forcedToolChoice) {
body.tool_choice = name ? { type: 'tool', name } : { type: 'any' };
forcing = true;
} else {
// Fable 5.1+, Opus 5.5 and Sonnet 5.5 answer 400 to forced tool use; the
// documented replacement is `auto` plus an explicit instruction.
systemNotes.push(
name
? `For this turn you must call the \`${name}\` tool. Begin your response with that tool call.`
: 'For this turn you must call one of the available tools. Begin your response with the tool call.',
);
}
}
}
// --- thinking ----------------------------------------------------
if (caps.reasoning !== false && profile.thinking !== 'none') {
const effort = req.reasoningEffort;
const adaptive: Json = profile.display ? { type: 'adaptive', display: 'summarized' } : { type: 'adaptive' };
const outputConfig: Json = {};
if (effort === 'none') {
if (profile.thinking === 'always') {
// Cannot be disabled: the lowest effort is the closest thing to "off".
body.thinking = adaptive;
outputConfig.effort = 'low';
} else if (profile.thinking === 'default_on') {
body.thinking = { type: profile.off ?? 'disabled' }; // `between_tools` takes no other field
}
// opt_in / budget models do not think unless asked.
} else if (profile.thinking === 'always' || profile.thinking === 'default_on') {
// `display: "summarized"` because these models hide the reasoning text by
// default, which would look like a long pause in the UI.
body.thinking = adaptive;
if (effort) outputConfig.effort = effort;
} else if (effort) {
// Forced tool use cannot be combined with thinking on the older generations.
if (profile.thinking === 'opt_in' && !forcing) body.thinking = adaptive;
if (profile.thinking === 'budget' && !forcing) {
const budget = Math.min(LEGACY_BUDGET[effort], maxTokens - 1_024); // must be < max_tokens, min 1024
if (budget >= 1_024) body.thinking = { type: 'enabled', budget_tokens: budget };
}
if (profile.effort) outputConfig.effort = effort;
}
if (Object.keys(outputConfig).length > 0) body.output_config = outputConfig;
}
// --- system prompt -----------------------------------------------
// There is no schema-less JSON mode (structured outputs need a schema), and
// assistant prefill is rejected by current models, so JSON mode is an instruction.
if (req.jsonMode) systemNotes.push('Respond with a single valid JSON object and nothing else — no prose, no Markdown code fences.');
const system = [req.system?.trim(), ...systemNotes].filter(Boolean).join('\n\n');
if (system) body.system = system;
body.messages = toWireMessages(req.messages, caps, opts.stripThinking);
if (opts.promptCache) body.cache_control = { type: 'ephemeral' }; // automatic caching of the reusable prefix
if (opts.useFallbacks) body.fallbacks = 'default';
return body;
}
/* ------------------------------------------------------------------ */
/* Adapter */
/* ------------------------------------------------------------------ */
function rootUrl(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1$/, '');
}
function isOfficial(cfg: ProviderConfig): boolean {
try {
return new URL(rootUrl(cfg)).hostname === 'api.anthropic.com';
} catch {
return false;
}
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig, betas: string[] = []): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an Anthropic API key in Settings to use Claude models.');
const h: Record<string, string> = { ...cfg.headers, 'x-api-key': cfg.apiKey, 'anthropic-version': API_VERSION };
const userBeta = Object.entries(cfg.headers ?? {}).find(([k]) => k.toLowerCase() === 'anthropic-beta')?.[1];
const all = [...(userBeta ? userBeta.split(',').map((s) => s.trim()) : []), ...betas].filter(Boolean);
for (const k of Object.keys(h)) if (k.toLowerCase() === 'anthropic-beta') delete h[k];
if (all.length > 0) h['anthropic-beta'] = [...new Set(all)].join(',');
return h;
}
function mapStopReason(reason: string): StopReason {
switch (reason) {
case 'end_turn':
case 'stop_sequence':
return 'end';
case 'tool_use':
return 'tool_use';
case 'max_tokens':
case 'model_context_window_exceeded':
return 'max_tokens';
case 'refusal':
return 'refusal';
default:
return 'other'; // pause_turn (server tools only) and anything new
}
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const profile = claudeProfile(req.model);
const official = isOfficial(cfg);
const opts: BuildOptions = {
official,
stripThinking: false,
// The beta-gated extras are only sent to api.anthropic.com; gateways and
// proxies frequently reject fields they do not know.
useFallbacks: official && profile.fallbacks,
eagerToolInput: official,
promptCache: official,
};
// Open the stream, degrading once per recognised 400 (see file header).
let http;
for (let attempt = 0; ; attempt++) {
http = await httpRequest(`${rootUrl(cfg)}/v1/messages`, {
...ctx,
headers: headers(cfg, opts.useFallbacks ? [FALLBACK_BETA] : []),
body: buildBody(req, profile, opts),
signal: req.signal,
});
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
const detail = err.providerMessage ?? '';
if (!opts.stripThinking && /signature/i.test(detail) && /thinking/i.test(detail)) opts.stripThinking = true;
else if (opts.useFallbacks && /fallback|anthropic-beta/i.test(detail)) opts.useFallbacks = false;
else if (opts.eagerToolInput && /eager_input_streaming/i.test(detail)) opts.eagerToolInput = false;
else if (opts.promptCache && /cache_control/i.test(detail)) opts.promptCache = false;
else throw err;
}
}
// Content blocks by index, accumulated exactly as the API defines them.
const blocks: Json[] = [];
const toolJson = new Map<number, string>();
let stopReason = '';
let sawStop = false;
let reasoningEmitted = false;
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
const readUsage = (u: unknown) => {
if (!isRecord(u)) return;
if (typeof u.input_tokens === 'number') usage.input = u.input_tokens;
if (typeof u.cache_read_input_tokens === 'number') usage.cacheRead = u.cache_read_input_tokens;
if (typeof u.cache_creation_input_tokens === 'number') usage.cacheWrite = u.cache_creation_input_tokens;
if (typeof u.output_tokens === 'number') usage.output = u.output_tokens;
};
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
switch (data.type) {
case 'message_start':
if (isRecord(data.message)) readUsage(data.message.usage);
break;
case 'content_block_start': {
const index = num(data.index);
const block: Json = isRecord(data.content_block) ? { ...data.content_block } : { type: 'unknown' };
blocks[index] = block;
if (block.type === 'tool_use') toolJson.set(index, '');
if (block.type === 'text' && typeof block.text === 'string' && block.text) yield { type: 'text', delta: block.text };
if (block.type === 'thinking' && reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
break;
}
case 'content_block_delta': {
const index = num(data.index);
const block = blocks[index];
const delta = data.delta;
if (!block || !isRecord(delta)) break;
if (delta.type === 'text_delta') {
const text = str(delta.text);
block.text = str(block.text) + text;
if (text) yield { type: 'text', delta: text };
} else if (delta.type === 'thinking_delta') {
const text = str(delta.thinking);
block.thinking = str(block.thinking) + text;
if (text) {
reasoningEmitted = true;
yield { type: 'reasoning', delta: text };
}
} else if (delta.type === 'signature_delta') {
block.signature = str(block.signature) + str(delta.signature);
} else if (delta.type === 'input_json_delta') {
toolJson.set(index, (toolJson.get(index) ?? '') + str(delta.partial_json));
} else if (delta.type === 'citations_delta') {
const list = Array.isArray(block.citations) ? block.citations : [];
list.push(delta.citation);
block.citations = list;
}
break;
}
case 'message_delta':
if (isRecord(data.delta) && typeof data.delta.stop_reason === 'string') stopReason = data.delta.stop_reason;
readUsage(data.usage);
break;
case 'message_stop':
sawStop = true;
break;
case 'error':
// e.g. overloaded_error delivered after the 200 response started
throw errorFromProvider(ctx, undefined, data);
default:
break; // ping, and event types added later
}
}
if (!sawStop && !stopReason) {
throw new AIError('network', 'The connection to Anthropic closed before the response finished.', { retryable: true });
}
// Tool calls are surfaced only now, at the end of the message, because the
// stop reason decides whether they are usable:
// - `refusal` can cut a tool_use off mid-input — never run that turn's tools;
// - `max_tokens` can truncate the last call — drop it rather than "repair" it;
// - after a mid-output fallback, tool_use blocks from before the last
// `fallback` block belong to the declined attempt and must not be echoed.
let boundary = -1;
blocks.forEach((b, i) => {
if (b?.type === 'fallback') boundary = i;
});
const content: ContentPart[] = [];
const replay: Json[] = [];
const calls: Array<{ id: string; name: string; input: Record<string, unknown> }> = [];
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
if (!block) continue;
const beforeBoundary = i < boundary;
if (block.type === 'text') {
const text = str(block.text);
if (!text) continue; // empty text blocks are rejected when echoed back
content.push({ type: 'text', text });
replay.push(block);
} else if (block.type === 'tool_use') {
if (beforeBoundary || stopReason === 'refusal') continue;
const raw = toolJson.get(i) ?? '';
const name = str(block.name);
let input = parseArgsStrict(raw);
if (!input) {
if (stopReason === 'max_tokens') continue; // truncated mid-input
input = parseToolArguments(raw, LABEL, name);
}
const call = { id: str(block.id), name, input };
calls.push(call);
content.push({ type: 'tool_call', ...call });
replay.push({ ...block, input });
} else if (block.type === 'fallback') {
replay.push(block); // must stay exactly where it appeared
} else if (!beforeBoundary) {
replay.push(block); // thinking, redacted_thinking, …: echoed back unchanged
}
}
for (const call of calls) yield { type: 'tool_call', call };
let mapped = mapStopReason(stopReason);
if (mapped === 'tool_use' && calls.length === 0) mapped = 'other';
if (mapped === 'end' && calls.length > 0) mapped = 'tool_use';
const message: ChatMessage = { role: 'assistant', content };
if (replay.length > 0) message.providerData = { kind: 'anthropic', data: { content: replay } };
yield {
type: 'done',
message,
stopReason: mapped,
// Anthropic reports uncached, cache-read and cache-write input separately.
usage: makeUsage({
input: usage.input + usage.cacheRead + usage.cacheWrite,
output: usage.output,
cacheRead: usage.cacheRead,
cacheWrite: usage.cacheWrite,
}),
};
}
function supported(caps: unknown, key: string): boolean | undefined {
if (!isRecord(caps) || !isRecord(caps[key])) return undefined;
const v = (caps[key] as Json).supported;
return typeof v === 'boolean' ? v : undefined;
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const ctx = errorContext(cfg);
const found: DiscoveredModel[] = [];
let afterId = '';
// https://platform.claude.com/docs/en/api/models/list — cursor pagination via has_more / last_id.
for (let page = 0; page < 20; page++) {
const url = `${rootUrl(cfg)}/v1/models?limit=1000${afterId ? `&after_id=${encodeURIComponent(afterId)}` : ''}`;
const res = await requestJson<Json>(url, { ...ctx, headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
for (const m of data) {
const id = str(m.id);
if (!id || !isChatModelId(id)) continue;
const model: DiscoveredModel = {
id,
// The API version in use rejects sampling parameters on every model.
capabilities: { tools: true, temperature: false },
};
if (typeof m.display_name === 'string') model.label = m.display_name;
if (typeof m.max_input_tokens === 'number') model.contextWindow = m.max_input_tokens;
if (typeof m.max_tokens === 'number') model.maxOutput = m.max_tokens;
const vision = supported(m.capabilities, 'image_input');
const pdf = supported(m.capabilities, 'pdf_input');
const thinking = supported(m.capabilities, 'thinking');
if (vision !== undefined) model.capabilities = { ...model.capabilities, vision };
if (pdf !== undefined) model.capabilities = { ...model.capabilities, pdf };
if (thinking !== undefined) model.capabilities = { ...model.capabilities, reasoning: thinking };
found.push(model);
}
if (res.has_more !== true || typeof res.last_id !== 'string' || !res.last_id) break;
afterId = res.last_id;
}
return withCatalogDefaults('anthropic', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
// Listing models is free and proves both connectivity and the key.
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const anthropicAdapter: ProviderAdapter = {
kind: 'anthropic',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://platform.claude.com/settings/keys',
keyHint: 'sk-ant-…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,206 @@
/**
* Static defaults for well-known models: capabilities, limits and list prices.
*
* Discovery APIs return ids (and sometimes limits) but almost never prices or
* feature flags, so `listModels` merges its results with this table. Unknown
* models simply get no defaults — nothing here is required for a model to work.
*
* Everything below was read from the official pages listed per provider on the
* dates noted. Prices are USD per million tokens, standard (non-batch,
* non-priority) tier, short-context tier where a provider has several. A field
* is omitted when it could not be confirmed from the page — do not guess.
*/
import type { DiscoveredModel, ModelCapabilities, ProviderKind } from '../types.ts';
export const CATALOG_UPDATED = '2026-11-10';
type Entry = Omit<DiscoveredModel, 'id' | 'capabilities'> & { capabilities?: Partial<ModelCapabilities> };
/* ------------------------------------------------------------------ */
/* OpenAI */
/* Prices: https://developers.openai.com/api/docs/pricing (fetched 2026-11-11)
/* Limits: https://developers.openai.com/api/docs/models (fetched 2026-11-11;
/* only the four current flagship models list limits there — the
/* limits on older models below are the long-published values and were
/* not re-fetched)
/* Reasoning models reject a custom `temperature`, hence temperature:false.
/* ------------------------------------------------------------------ */
const OA_REASONING: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
const OA_CLASSIC: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: false, temperature: true };
const OA_FLAGSHIP_LIMITS = { contextWindow: 1_050_000, maxOutput: 128_000 };
const OPENAI: Record<string, Entry> = {
'gpt-6-astra': { label: 'GPT-6 Astra', ...OA_FLAGSHIP_LIMITS, priceIn: 10, priceOut: 50, capabilities: OA_REASONING },
'gpt-6.1-sol': { label: 'GPT-6.1 Sol', ...OA_FLAGSHIP_LIMITS, priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
'gpt-6.1-terra': { label: 'GPT-6.1 Terra', ...OA_FLAGSHIP_LIMITS, priceIn: 1, priceOut: 5, capabilities: OA_REASONING },
'gpt-6-luna': { label: 'GPT-6 Luna', ...OA_FLAGSHIP_LIMITS, priceIn: 0.1, priceOut: 0.5, capabilities: OA_REASONING },
'gpt-6-sol': { label: 'GPT-6 Sol', priceIn: 2, priceOut: 10, capabilities: OA_REASONING },
'gpt-5.6-sol': { label: 'GPT-5.6 Sol', priceIn: 4, priceOut: 20, capabilities: OA_REASONING },
'gpt-5.6-terra': { label: 'GPT-5.6 Terra', priceIn: 2, priceOut: 12, capabilities: OA_REASONING },
'gpt-5.6-luna': { label: 'GPT-5.6 Luna', priceIn: 0.2, priceOut: 1.2, capabilities: OA_REASONING },
'gpt-5.5': { label: 'GPT-5.5', priceIn: 5, priceOut: 30, capabilities: OA_REASONING },
'gpt-5.5-pro': { label: 'GPT-5.5 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
'gpt-5.4': { label: 'GPT-5.4', priceIn: 2.5, priceOut: 15, capabilities: OA_REASONING },
'gpt-5.4-pro': { label: 'GPT-5.4 Pro', priceIn: 30, priceOut: 180, capabilities: OA_REASONING },
'gpt-5.4-mini': { label: 'GPT-5.4 mini', priceIn: 0.75, priceOut: 4.5, capabilities: OA_REASONING },
'gpt-5.4-nano': { label: 'GPT-5.4 nano', priceIn: 0.2, priceOut: 1.25, capabilities: OA_REASONING },
'gpt-5.3-codex': { label: 'GPT-5.3 Codex', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
'gpt-5.2': { label: 'GPT-5.2', priceIn: 1.75, priceOut: 14, capabilities: OA_REASONING },
'gpt-5.2-pro': { label: 'GPT-5.2 Pro', priceIn: 21, priceOut: 168, capabilities: OA_REASONING },
'gpt-5.1': { label: 'GPT-5.1', priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
'gpt-5': { label: 'GPT-5', contextWindow: 400_000, maxOutput: 128_000, priceIn: 1.25, priceOut: 10, capabilities: OA_REASONING },
'gpt-5-mini': { label: 'GPT-5 mini', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.25, priceOut: 2, capabilities: OA_REASONING },
'gpt-5-nano': { label: 'GPT-5 nano', contextWindow: 400_000, maxOutput: 128_000, priceIn: 0.05, priceOut: 0.4, capabilities: OA_REASONING },
'gpt-5-pro': { label: 'GPT-5 Pro', priceIn: 15, priceOut: 120, capabilities: OA_REASONING },
'chat-latest': { label: 'ChatGPT (latest)', priceIn: 5, priceOut: 30, capabilities: OA_CLASSIC },
'gpt-4.1': { label: 'GPT-4.1', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 2, priceOut: 8, capabilities: OA_CLASSIC },
'gpt-4.1-mini': { label: 'GPT-4.1 mini', contextWindow: 1_047_576, maxOutput: 32_768, priceIn: 0.4, priceOut: 1.6, capabilities: OA_CLASSIC },
'gpt-4o': { label: 'GPT-4o', contextWindow: 128_000, maxOutput: 16_384, priceIn: 2.5, priceOut: 10, capabilities: OA_CLASSIC },
'gpt-4o-mini': { label: 'GPT-4o mini', contextWindow: 128_000, maxOutput: 16_384, priceIn: 0.15, priceOut: 0.6, capabilities: OA_CLASSIC },
o3: { label: 'o3', contextWindow: 200_000, maxOutput: 100_000, priceIn: 2, priceOut: 8, capabilities: OA_REASONING },
'o3-pro': { label: 'o3-pro', contextWindow: 200_000, maxOutput: 100_000, priceIn: 20, priceOut: 80, capabilities: OA_REASONING },
};
/* ------------------------------------------------------------------ */
/* Anthropic */
/* Prices: https://platform.claude.com/docs/en/about-claude/pricing (fetched 2026-11-11)
/* Ids and limits: https://platform.claude.com/docs/en/about-claude/models/overview
/* (via Anthropic's Claude API reference, cached 2026-11-03)
/* temperature:false for every Claude model: the adapter speaks API version
/* 2026-10-21, which rejects temperature/top_p/top_k on all models
/* (https://platform.claude.com/docs/en/api/migration-2026-10-21).
/* Claude Haiku 5.5 prices are for prompts up to 100K tokens ($0.50/$2.50 above).
/* ------------------------------------------------------------------ */
const CLAUDE: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: false };
const CLAUDE_1M = { contextWindow: 1_000_000, maxOutput: 128_000 };
const ANTHROPIC: Record<string, Entry> = {
'claude-fable-5-2': { label: 'Claude Fable 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
'claude-mythos-5-2': { label: 'Claude Mythos 5.2', ...CLAUDE_1M, priceIn: 8, priceOut: 40, capabilities: CLAUDE },
'claude-fable-5-1': { label: 'Claude Fable 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-mythos-5-1': { label: 'Claude Mythos 5.1', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-fable-5': { label: 'Claude Fable 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-mythos-5': { label: 'Claude Mythos 5', ...CLAUDE_1M, priceIn: 10, priceOut: 50, capabilities: CLAUDE },
'claude-opus-5-5': { label: 'Claude Opus 5.5', ...CLAUDE_1M, priceIn: 4, priceOut: 20, capabilities: CLAUDE },
'claude-opus-5': { label: 'Claude Opus 5', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-8': { label: 'Claude Opus 4.8', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-7': { label: 'Claude Opus 4.7', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-6': { label: 'Claude Opus 4.6', ...CLAUDE_1M, priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-opus-4-5': { label: 'Claude Opus 4.5', priceIn: 5, priceOut: 25, capabilities: CLAUDE },
'claude-sonnet-5-5': { label: 'Claude Sonnet 5.5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
'claude-sonnet-5': { label: 'Claude Sonnet 5', ...CLAUDE_1M, priceIn: 2, priceOut: 10, capabilities: CLAUDE },
'claude-sonnet-4-6': { label: 'Claude Sonnet 4.6', ...CLAUDE_1M, priceIn: 3, priceOut: 15, capabilities: CLAUDE },
'claude-sonnet-4-5': { label: 'Claude Sonnet 4.5', priceIn: 3, priceOut: 15, capabilities: CLAUDE },
'claude-haiku-5-5': { label: 'Claude Haiku 5.5', ...CLAUDE_1M, priceIn: 0.1, priceOut: 0.5, capabilities: CLAUDE },
'claude-haiku-4-5': { label: 'Claude Haiku 4.5', contextWindow: 200_000, maxOutput: 64_000, priceIn: 1, priceOut: 5, capabilities: CLAUDE },
};
/* ------------------------------------------------------------------ */
/* Google Gemini */
/* Prices: https://ai.google.dev/gemini-api/docs/pricing (fetched 2026-11-11,
/* now served from https://aistudio.google.com/docs/pricing)
/* Limits: https://ai.google.dev/gemini-api/docs/models (fetched 2026-11-11;
/* only the Gemini 4 models list limits — `GET /v1beta/models`
/* supplies inputTokenLimit/outputTokenLimit for the rest at runtime)
/* Scheduled changes announced on the pricing page (NOT applied here):
/* - gemini-4-carbon / gemini-4-argon: $4 / $20 from 2026-12-01
/* - gemini-3.8-flash / gemini-3.6-flash: $1.50 / $7.50 from 2027-01-01
/* - gemini-2.5-pro / gemini-2.5-flash are deprecated and are routed to
/* gemini-3.8-flash / gemini-3.5-flash-lite (and billed as such) from 2026-12-01
/* Pro-tier prices are for prompts up to 200K tokens.
/* ------------------------------------------------------------------ */
const GEMINI_CAPS: Partial<ModelCapabilities> = { tools: true, vision: true, pdf: true, reasoning: true, temperature: true };
const GEMINI_4_LIMITS = { contextWindow: 1_048_576, maxOutput: 65_536 };
const GEMINI: Record<string, Entry> = {
'gemini-4-carbon': { label: 'Gemini 4 Carbon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-4-argon': { label: 'Gemini 4 Argon', ...GEMINI_4_LIMITS, priceIn: 2, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-3.8-flash': { label: 'Gemini 3.8 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
'gemini-3.6-flash': { label: 'Gemini 3.6 Flash', priceIn: 0.75, priceOut: 3.75, capabilities: GEMINI_CAPS },
'gemini-3.5-flash-lite': { label: 'Gemini 3.5 Flash-Lite', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
'gemini-3.1-pro-preview': { label: 'Gemini 3.1 Pro (preview)', priceIn: 2, priceOut: 12, capabilities: GEMINI_CAPS },
'gemini-3.1-flash-lite': { label: 'Gemini 3.1 Flash-Lite', priceIn: 0.25, priceOut: 1.5, capabilities: GEMINI_CAPS },
'gemini-3-flash-preview': { label: 'Gemini 3 Flash (preview)', priceIn: 0.5, priceOut: 3, capabilities: GEMINI_CAPS },
'gemini-2.5-pro': { label: 'Gemini 2.5 Pro', priceIn: 1.25, priceOut: 10, capabilities: GEMINI_CAPS },
'gemini-2.5-flash': { label: 'Gemini 2.5 Flash', priceIn: 0.3, priceOut: 2.5, capabilities: GEMINI_CAPS },
'gemini-2.5-flash-lite': { label: 'Gemini 2.5 Flash-Lite', priceIn: 0.1, priceOut: 0.4, capabilities: GEMINI_CAPS },
};
const TABLES: Record<ProviderKind, Record<string, Entry>> = {
openai: OPENAI,
anthropic: ANTHROPIC,
gemini: GEMINI,
// Aggregators and local servers host arbitrary models at their own prices.
openai_compatible: {},
ollama: {},
};
/**
* What may follow a known id for it to still be "the same model at the same
* price": a dated snapshot or a moving alias, e.g. `gpt-5-2025-08-07`,
* `claude-haiku-4-5-20251001`, `claude-opus-4-5@20251101`, `gemini-2.5-flash-latest`.
*/
const SNAPSHOT_SUFFIX = /^[-@:](\d{4}-\d{2}-\d{2}|\d{6,8}|\d{3,4}|latest|preview|preview-\d{2}-\d{2,4}|v\d+(:\d+)?)$/;
function clone(e: Entry): Partial<DiscoveredModel> {
return { ...e, ...(e.capabilities ? { capabilities: { ...e.capabilities } } : {}) };
}
function lookupIn(table: Record<string, Entry>, id: string): Partial<DiscoveredModel> {
const exact = table[id];
if (exact) return clone(exact);
// Longest known id that is a prefix of `id` and ends on a separator.
let best = '';
for (const key of Object.keys(table)) {
if (key.length > best.length && id.startsWith(key) && /[-@:]/.test(id.charAt(key.length))) best = key;
}
if (!best) return {};
const entry = table[best];
if (!entry) return {};
const out = clone(entry);
delete out.label; // the label names the base model, not this variant
if (!SNAPSHOT_SUFFIX.test(id.slice(best.length))) {
// A genuinely different variant (`o3-deep-research`, `gpt-4o-audio-...`):
// capabilities are a fair guess, the price is not.
delete out.priceIn;
delete out.priceOut;
}
return out;
}
/**
* Defaults for a model: exact id first, then the longest known id that prefixes
* it (so dated snapshots match their base model). Returns `{}` when unknown.
*/
export function lookupModelDefaults(kind: ProviderKind, modelId: string): Partial<DiscoveredModel> {
// Gemini's listing names models "models/<id>".
const id = modelId.trim().replace(/^models\//, '');
if (!id) return {};
if (kind === 'openai_compatible') {
// Routers name models "<vendor>/<model>" (`anthropic/claude-sonnet-5-5`).
// Borrow capabilities and limits from the vendor's table, never its prices.
const bare = id.slice(id.lastIndexOf('/') + 1).replace(/:[a-z]+$/, '');
for (const table of [OPENAI, ANTHROPIC, GEMINI]) {
const hit = lookupIn(table, bare);
if (Object.keys(hit).length > 0) {
delete hit.priceIn;
delete hit.priceOut;
// Temperature limits are properties of the first-party APIs, not of the router.
if (hit.capabilities) delete hit.capabilities.temperature;
return hit;
}
}
return {};
}
return lookupIn(TABLES[kind], id);
}
/** Every catalogued model id for a provider (used as a fallback list when discovery is unavailable). */
export function catalogModelIds(kind: ProviderKind): string[] {
return Object.keys(TABLES[kind]);
}

View file

@ -0,0 +1,138 @@
/**
* Helpers shared by the adapters that are about our neutral contract rather
* than about HTTP: tool-argument parsing, model-list post-processing, the
* connection-test wrapper.
*/
import { jsonrepair } from 'jsonrepair';
import { AIError, type DiscoveredModel, type ProviderKind, type TestResult, type Usage } from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { isRecord } from './http.ts';
/** Strict parse of tool-call arguments; `undefined` when they are not a JSON object. */
export function parseArgsStrict(raw: string): Record<string, unknown> | undefined {
const text = raw.trim();
if (text === '') return {};
try {
const value: unknown = JSON.parse(text);
return isRecord(value) ? value : undefined;
} catch {
return undefined;
}
}
/**
* Parses the JSON arguments of a completed tool call. Models occasionally emit
* slightly broken JSON (trailing commas, unescaped newlines); `jsonrepair`
* fixes those. Anything still unparseable is an `invalid_response`.
*
* Do NOT use this on a call that was cut off by the output-token limit —
* repairing a truncated call would silently invent a shorter input.
*/
export function parseToolArguments(raw: unknown, provider: string, toolName: string): Record<string, unknown> {
if (isRecord(raw)) return raw; // some servers send an object instead of a JSON string
const text = typeof raw === 'string' ? raw : raw == null ? '' : String(raw);
const strict = parseArgsStrict(text);
if (strict) return strict;
try {
const repaired: unknown = JSON.parse(jsonrepair(text));
if (isRecord(repaired)) return repaired;
} catch {
/* fall through */
}
throw new AIError('invalid_response', `${provider} produced a tool call for "${toolName}" with arguments that are not valid JSON.`, {
providerMessage: text.length > 300 ? `${text.slice(0, 300)}…` : text,
});
}
/** Stable id for providers that do not issue tool-call ids themselves. */
export function newToolCallId(prefix: string): string {
return `${prefix}_${globalThis.crypto.randomUUID().replace(/-/g, '').slice(0, 24)}`;
}
export function dataUrl(mime: string, base64: string): string {
return `data:${mime};base64,${base64}`;
}
export function num(v: unknown): number {
return typeof v === 'number' && Number.isFinite(v) ? v : 0;
}
export function str(v: unknown): string {
return typeof v === 'string' ? v : '';
}
/**
* Usage convention used by every adapter:
* inputTokens = ALL prompt tokens, including cached ones
* cacheRead/cacheWriteTokens = the cached subsets of inputTokens
* outputTokens = ALL generated tokens, including reasoning
* reasoningTokens = the reasoning subset of outputTokens
*/
export function makeUsage(u: { input: number; output: number; cacheRead?: number; cacheWrite?: number; reasoning?: number }): Usage {
const usage: Usage = { inputTokens: u.input, outputTokens: u.output };
if (u.cacheRead) usage.cacheReadTokens = u.cacheRead;
if (u.cacheWrite) usage.cacheWriteTokens = u.cacheWrite;
if (u.reasoning) usage.reasoningTokens = u.reasoning;
return usage;
}
/**
* Ids that are obviously not text-chat models: embeddings, speech, image and
* video generation, moderation, realtime/audio-only, rerankers, legacy
* completion-only models.
*/
const NON_CHAT_RE = new RegExp(
[
'embed', // text-embedding-3-large, gemini-embedding-001, nomic-embed-text
'\\btts\\b|whisper|transcri|speech|\\baudio\\b|realtime|\\blive\\b',
'dall-e|gpt-image|image-generation|imagen|-image\\b|image-preview|\\bsora\\b|\\bveo\\b|lyria|stable-diffusion',
'moderation|-guard\\b|rerank|\\baqa\\b',
'computer-use|robotics|deep-research',
'^(babbage|davinci|curie|ada)\\b|^text-|turbo-instruct',
].join('|'),
'i',
);
export function isChatModelId(id: string): boolean {
return !NON_CHAT_RE.test(id);
}
/** Merges discovery results with catalog defaults; values the provider reported always win. */
export function withCatalogDefaults(kind: ProviderKind, models: DiscoveredModel[]): DiscoveredModel[] {
const seen = new Set<string>();
const out: DiscoveredModel[] = [];
for (const m of models) {
if (!m.id || seen.has(m.id)) continue;
seen.add(m.id);
const d = lookupModelDefaults(kind, m.id);
const merged: DiscoveredModel = { ...d, ...stripUndefined(m), id: m.id };
if (d.capabilities || m.capabilities) merged.capabilities = { ...d.capabilities, ...stripUndefined(m.capabilities ?? {}) };
out.push(merged);
}
return out;
}
function stripUndefined<T extends object>(obj: T): T {
const out = {} as Record<string, unknown>;
for (const [k, v] of Object.entries(obj)) if (v !== undefined && v !== null) out[k] = v;
return out as T;
}
/** Shared implementation of `ProviderAdapter.test()`: never throws, always friendly. */
export async function runTest(label: string, check: () => Promise<{ modelCount?: number; note?: string }>): Promise<TestResult> {
const started = Date.now();
try {
const { modelCount, note } = await check();
const latencyMs = Date.now() - started;
const message =
note ?? (modelCount !== undefined ? `Connected to ${label} — ${modelCount} model${modelCount === 1 ? '' : 's'} available.` : `Connected to ${label}.`);
return modelCount !== undefined ? { ok: true, message, latencyMs, modelCount } : { ok: true, message, latencyMs };
} catch (err) {
const latencyMs = Date.now() - started;
const message = err instanceof AIError ? err.message : `Could not connect to ${label}.`;
return { ok: false, message, latencyMs };
}
}
/** Timeout used by `test()` so a dead endpoint fails fast instead of hanging the settings page. */
export const TEST_TIMEOUT_MS = 20_000;

View file

@ -0,0 +1,342 @@
/**
* Google Gemini — `models/{model}:streamGenerateContent?alt=sse`.
* https://ai.google.dev/api/generate-content
* https://ai.google.dev/gemini-api/docs/function-calling
* https://ai.google.dev/gemini-api/docs/thinking
* https://ai.google.dev/gemini-api/docs/thought-signatures
*
* Quirks handled here:
* - The API key travels in the `x-goog-api-key` header, never in the URL.
* - Function declarations accept only a subset of JSON Schema, so schemas are
* down-converted (`toGeminiSchema`).
* - Gemini issues no tool-call ids; stable ones are synthesised.
* - Thinking models attach a `thoughtSignature` to parts (notably function calls)
* that must be sent back unchanged, so each turn's raw parts live in `providerData`.
* - Thinking is configured with `thinkingLevel` on Gemini 3 and later and with a
* numeric `thinkingBudget` on 2.5; an unsupported config is dropped and retried.
* - A response can be blocked (`promptFeedback.blockReason`, finishReason SAFETY).
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { isChatModelId, makeUsage, newToolCallId, num, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'Google Gemini';
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com';
function root(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/v1(beta)?$/, '');
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add a Gemini API key in Settings to use Gemini models.');
return { ...cfg.headers, 'x-goog-api-key': cfg.apiKey };
}
const SCHEMA_KEYS = new Set(['type', 'description', 'enum', 'properties', 'required', 'items', 'nullable', 'format', 'minimum', 'maximum', 'minItems', 'maxItems', 'anyOf', 'title']);
const SCHEMA_FORMATS = new Set(['date-time', 'enum', 'int32', 'int64', 'float', 'double']);
/** Reduces JSON Schema to the OpenAPI-style subset Gemini's `parameters` field accepts. */
export function toGeminiSchema(schema: unknown): Json {
if (!isRecord(schema)) return { type: 'string' };
const out: Json = {};
for (const [key, value] of Object.entries(schema)) {
if (!SCHEMA_KEYS.has(key)) continue; // additionalProperties, $schema, default, pattern, const, ...
if (key === 'properties' && isRecord(value)) {
out.properties = Object.fromEntries(Object.entries(value).map(([k, v]) => [k, toGeminiSchema(v)]));
} else if (key === 'items') {
out.items = toGeminiSchema(value);
} else if (key === 'anyOf' && Array.isArray(value)) {
out.anyOf = value.map(toGeminiSchema);
} else if (key === 'type' && Array.isArray(value)) {
// ["string","null"] -> nullable string
const real = value.filter((t) => t !== 'null');
out.type = real[0] ?? 'string';
if (value.includes('null')) out.nullable = true;
} else if (key === 'format') {
if (typeof value === 'string' && SCHEMA_FORMATS.has(value)) out.format = value;
} else {
out[key] = value;
}
}
if (out.enum && !out.type) out.type = 'string';
if (out.type === 'object' && !out.properties) out.properties = {};
return out;
}
function userParts(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const out: Json[] = [];
for (const p of parts) {
if (p.type === 'tool_result') {
out.push({ functionResponse: { name: p.name, response: p.isError ? { error: p.output } : { result: p.output } } });
} else if (p.type === 'text' && p.text) {
out.push({ text: p.text });
} else if (p.type === 'image') {
if (caps.vision === false) out.push({ text: '[An image was attached, but this model cannot read images.]' });
else out.push({ inlineData: { mimeType: p.mime, data: p.data } });
} else if (p.type === 'file') {
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ inlineData: { mimeType: p.mime, data: p.data } });
else if (p.mime.startsWith('text/')) out.push({ text: Buffer.from(p.data, 'base64').toString('utf8') });
else out.push({ text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
return out;
}
function toContents(messages: ChatMessage[], caps: Partial<ModelCapabilities>): Json[] {
const contents: Json[] = [];
for (const msg of messages) {
let parts: Json[];
if (msg.role === 'assistant') {
const pd = msg.providerData;
if (pd?.kind === 'gemini' && isRecord(pd.data) && Array.isArray(pd.data.parts)) {
parts = pd.data.parts.filter(isRecord); // thought signatures included, exactly as received
} else {
parts = [];
for (const p of msg.content) {
if (p.type === 'text' && p.text) parts.push({ text: p.text });
else if (p.type === 'tool_call') parts.push({ functionCall: { name: p.name, args: p.input ?? {} } });
}
}
} else {
parts = userParts(msg.content, caps);
}
if (parts.length === 0) continue;
const role = msg.role === 'assistant' ? 'model' : 'user';
const last = contents[contents.length - 1];
// Gemini requires strictly alternating roles; merge consecutive turns of the same role.
if (last && last.role === role) (last.parts as Json[]).push(...parts);
else contents.push({ role, parts });
}
if (contents[0]?.role === 'model') contents.unshift({ role: 'user', parts: [{ text: '(continuing the conversation)' }] });
return contents;
}
type ThinkingStyle = 'level' | 'budget' | 'off';
function thinkingStyleFor(model: string): ThinkingStyle {
const m = /gemini-(\d+)(?:\.(\d+))?/.exec(model);
if (!m) return 'level';
const major = Number(m[1]);
if (major >= 3) return 'level';
if (major === 2 && Number(m[2] ?? 0) >= 5) return 'budget';
return 'off';
}
const BUDGETS = { none: 0, low: 2_048, medium: 8_192, high: 24_576 } as const;
function buildBody(req: ChatRequest, thinking: ThinkingStyle, jsonMode: boolean): Json {
const caps = req.capabilities ?? {};
const body: Json = { contents: toContents(req.messages, caps) };
if (req.system?.trim()) body.systemInstruction = { parts: [{ text: req.system.trim() }] };
const hasTools = caps.tools !== false && (req.tools?.length ?? 0) > 0;
if (hasTools) {
body.tools = [{ functionDeclarations: req.tools!.map((t) => ({ name: t.name, description: t.description, parameters: toGeminiSchema({ type: 'object', ...t.parameters }) })) }];
const choice = req.toolChoice;
if (choice === 'none') body.toolConfig = { functionCallingConfig: { mode: 'NONE' } };
else if (choice === 'required') body.toolConfig = { functionCallingConfig: { mode: 'ANY' } };
else if (isRecord(choice)) body.toolConfig = { functionCallingConfig: { mode: 'ANY', allowedFunctionNames: [choice.name] } };
}
const gen: Json = {};
if (req.temperature !== undefined && caps.temperature !== false) gen.temperature = req.temperature;
if (req.maxOutputTokens) gen.maxOutputTokens = req.maxOutputTokens;
// JSON mode cannot be combined with function calling on most models.
if (jsonMode && req.jsonMode && !hasTools) gen.responseMimeType = 'application/json';
if (caps.reasoning !== false && thinking !== 'off') {
const effort = req.reasoningEffort;
if (thinking === 'level') {
const cfg: Json = { includeThoughts: true };
if (effort) cfg.thinkingLevel = effort === 'none' ? 'minimal' : effort;
gen.thinkingConfig = cfg;
} else if (effort) {
// 2.5 Pro cannot switch thinking off; its minimum budget is 128 tokens.
const budget = effort === 'none' && /pro/.test(req.model) ? 128 : BUDGETS[effort];
gen.thinkingConfig = { thinkingBudget: budget, includeThoughts: budget > 0 };
} else {
gen.thinkingConfig = { includeThoughts: true };
}
}
if (Object.keys(gen).length > 0) body.generationConfig = gen;
return body;
}
function mapFinish(reason: string): StopReason {
switch (reason) {
case 'STOP':
return 'end';
case 'MAX_TOKENS':
return 'max_tokens';
case 'SAFETY':
case 'RECITATION':
case 'BLOCKLIST':
case 'PROHIBITED_CONTENT':
case 'SPII':
case 'IMAGE_SAFETY':
return 'refusal';
default:
return 'other'; // MALFORMED_FUNCTION_CALL, OTHER, UNEXPECTED_TOOL_CALL, ...
}
}
function modelPath(model: string): string {
return model.startsWith('models/') || model.startsWith('tunedModels/') ? model : `models/${model}`;
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const url = `${root(cfg)}/v1beta/${modelPath(req.model)}:streamGenerateContent?alt=sse`;
let thinking = thinkingStyleFor(req.model);
let jsonMode = true;
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(url, { ...ctx, headers: headers(cfg), body: buildBody(req, thinking, jsonMode), signal: req.signal });
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 3) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (thinking !== 'off' && /thinking|thought/.test(detail)) thinking = thinking === 'level' ? 'budget' : 'off';
else if (jsonMode && req.jsonMode && /response_?mime|json/.test(detail)) jsonMode = false;
else throw err;
}
}
const parts: Json[] = [];
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
let text = '';
let finish = '';
let blocked = '';
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
if (isRecord(data.promptFeedback) && str(data.promptFeedback.blockReason)) blocked = str(data.promptFeedback.blockReason);
if (isRecord(data.usageMetadata)) {
const u = data.usageMetadata;
usage.input = num(u.promptTokenCount) || usage.input;
usage.reasoning = num(u.thoughtsTokenCount) || usage.reasoning;
// candidatesTokenCount excludes thinking; both are billed as output.
usage.output = num(u.candidatesTokenCount) + num(u.thoughtsTokenCount) || usage.output;
usage.cacheRead = num(u.cachedContentTokenCount) || usage.cacheRead;
}
const candidate = Array.isArray(data.candidates) ? data.candidates[0] : undefined;
if (!isRecord(candidate)) continue;
if (str(candidate.finishReason)) finish = str(candidate.finishReason);
const chunkParts = isRecord(candidate.content) && Array.isArray(candidate.content.parts) ? candidate.content.parts.filter(isRecord) : [];
for (const part of chunkParts) {
if (isRecord(part.functionCall)) {
const call = { id: str(part.functionCall.id) || newToolCallId('gem'), name: str(part.functionCall.name), input: isRecord(part.functionCall.args) ? part.functionCall.args : {} };
calls.push(call);
parts.push(part);
yield { type: 'tool_call', call };
} else if (typeof part.text === 'string') {
if (part.thought === true) {
if (part.text) yield { type: 'reasoning', delta: part.text };
// Thought summaries are display-only; only a signature needs to survive.
if (part.thoughtSignature) parts.push({ text: '', thought: true, thoughtSignature: part.thoughtSignature });
} else {
if (part.text) {
text += part.text;
yield { type: 'text', delta: part.text };
}
// Merge streamed text fragments, but keep a part that carries a signature intact.
const last = parts[parts.length - 1];
if (last && typeof last.text === 'string' && last.thought !== true && !last.thoughtSignature && !part.thoughtSignature) last.text += part.text;
else parts.push({ ...part });
}
} else {
parts.push(part); // inlineData, executableCode, ... replayed untouched
}
}
}
if (blocked && !text && calls.length === 0) {
throw new AIError('content_filter', 'Gemini blocked this request under its safety policy.', { providerMessage: `blockReason: ${blocked}` });
}
if (!finish && !text && calls.length === 0) {
throw new AIError('network', 'The connection to Gemini closed before the response finished.', { retryable: true });
}
if (finish === 'MALFORMED_FUNCTION_CALL' && calls.length === 0) {
throw new AIError('invalid_response', 'Gemini produced a malformed tool call. Try again.', { retryable: true });
}
const content: ContentPart[] = [];
if (text) content.push({ type: 'text', text });
for (const call of calls) content.push({ type: 'tool_call', ...call });
const message: ChatMessage = { role: 'assistant', content };
const replay = parts.filter((p) => !(typeof p.text === 'string' && p.text === '' && !p.thoughtSignature));
if (replay.length > 0) message.providerData = { kind: 'gemini', data: { parts: replay } };
yield {
type: 'done',
message,
stopReason: calls.length > 0 ? 'tool_use' : mapFinish(finish || 'STOP'),
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
};
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const found: DiscoveredModel[] = [];
let pageToken = '';
for (let page = 0; page < 10; page += 1) {
const url = `${root(cfg)}/v1beta/models?pageSize=1000${pageToken ? `&pageToken=${encodeURIComponent(pageToken)}` : ''}`;
const res = await requestJson<Json>(url, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
for (const m of Array.isArray(res.models) ? res.models.filter(isRecord) : []) {
const id = str(m.name).replace(/^models\//, '');
const methods = Array.isArray(m.supportedGenerationMethods) ? m.supportedGenerationMethods.map(String) : [];
if (!id || !methods.includes('generateContent') || !isChatModelId(id)) continue;
if (!/^(gemini|gemma|learnlm)/.test(id)) continue;
const model: DiscoveredModel = { id };
if (str(m.displayName)) model.label = str(m.displayName);
if (num(m.inputTokenLimit)) model.contextWindow = num(m.inputTokenLimit);
if (num(m.outputTokenLimit)) model.maxOutput = num(m.outputTokenLimit);
if (typeof m.thinking === 'boolean') model.capabilities = { reasoning: m.thinking };
if (id.startsWith('gemma')) model.capabilities = { ...model.capabilities, tools: false, pdf: false };
found.push(model);
}
pageToken = str(res.nextPageToken);
if (!pageToken) break;
}
return withCatalogDefaults('gemini', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const geminiAdapter: ProviderAdapter = {
kind: 'gemini',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://aistudio.google.com/apikey',
keyHint: 'AIza…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

519
server/ai/providers/http.ts Normal file
View file

@ -0,0 +1,519 @@
/**
* Shared HTTP plumbing for provider adapters: a fetch wrapper with timeouts,
* an SSE parser over Web Streams, and one place that turns provider failures
* into `AIError`s. Uses only global `fetch` — no SDKs.
*
* Security rule for this whole folder: an API key must never reach an error
* message, a log line or a thrown value. Everything user-visible that contains
* provider text goes through `scrubSecrets` first.
*/
import { AIError, type AIErrorCode } from '../types.ts';
/* ------------------------------------------------------------------ */
/* Secret scrubbing */
/* ------------------------------------------------------------------ */
const KEY_PATTERNS: RegExp[] = [
/\bsk-[A-Za-z0-9_-]{8,}/g, // OpenAI / Anthropic / OpenRouter style
/\bAIza[0-9A-Za-z_-]{16,}/g, // Google API keys
/\bgsk_[A-Za-z0-9]{8,}/g, // Groq
/(Bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
/([?&](?:key|api_key|apikey|access_token)=)[^&\s"']+/gi,
];
/** Removes known secrets (and anything that looks like an API key) from provider text. */
export function scrubSecrets(text: string, secrets: Array<string | null | undefined> = []): string {
let out = text;
for (const s of secrets) {
if (s && s.length >= 4) out = out.split(s).join('[redacted]');
}
for (const re of KEY_PATTERNS) {
out = out.replace(re, (_m, prefix?: unknown) => (typeof prefix === 'string' ? `${prefix}[redacted]` : '[redacted]'));
}
return out;
}
function clip(text: string, max = 800): string {
const t = text.trim();
return t.length > max ? `${t.slice(0, max)}…` : t;
}
/* ------------------------------------------------------------------ */
/* Error mapping */
/* ------------------------------------------------------------------ */
export interface ErrorContext {
/** Human provider label used in messages ("OpenAI", "Ollama"). */
provider: string;
/** Values that must never appear in an error (API key, custom header values). */
secrets?: Array<string | null | undefined>;
}
interface ProviderErrorFields {
message: string;
type: string;
code: string;
/** Whole body as text, used for keyword matching (quota ids, reasons, ...). */
raw: string;
}
/** Pulls message/type/code out of the error envelopes used by the providers we talk to. */
export function extractProviderError(body: unknown): ProviderErrorFields {
let node: unknown = body;
if (typeof node === 'string') {
const text = node;
try {
node = JSON.parse(text);
} catch {
return { message: text.trim().startsWith('<') ? '' : text.trim(), type: '', code: '', raw: text };
}
}
const raw = safeStringify(node);
// Gemini sometimes wraps the error object in an array.
if (Array.isArray(node)) node = node[0];
if (!isRecord(node)) return { message: '', type: '', code: '', raw };
const err = node.error ?? node;
// Ollama: {"error": "model 'x' not found"}
if (typeof err === 'string') return { message: err, type: '', code: '', raw };
if (!isRecord(err)) return { message: '', type: '', code: '', raw };
const message = typeof err.message === 'string' ? err.message : typeof node.message === 'string' ? node.message : '';
// OpenAI: error.type + error.code; Anthropic: error.type; Gemini: error.status (+ numeric code).
const type = [err.type, err.status].filter((v): v is string => typeof v === 'string').join(' ');
const code = typeof err.code === 'string' ? err.code : '';
return { message, type, code, raw };
}
const CONTEXT_RE =
/context[_ ]length|context[_ ]window|maximum context|max(?:imum)? (?:input )?tokens? (?:limit|allowed)|too many (?:input )?tokens|prompt is too long|input is too long|input token count|exceeds? the (?:model'?s? )?(?:maximum|max)[^.]{0,40}tokens|reduce the length|request[_ ]too[_ ]large|exceed_context_size|n_ctx/i;
const QUOTA_RE =
/insufficient[_ ]quota|billing|credit balance|exceeded your current quota|out of credits|insufficient (?:credits|funds|balance)|payment required|spend(?:ing)? limit/i;
const AUTH_RE = /invalid[_ ]api[_ ]key|incorrect api key|api key not valid|api_key_invalid|authentication[_ ]error|invalid x-api-key|unauthorized|no auth credentials/i;
const FILTER_RE = /content[_ ]filter|content[_ ]policy|content management policy|safety system|responsible ai/i;
function classify(status: number | undefined, f: ProviderErrorFields): AIErrorCode {
const hay = `${f.type} ${f.code} ${f.message}`;
const s = status ?? 0;
// Provider error names win over the status code: they are also all we have
// for errors delivered inside an already-open (HTTP 200) stream.
if (/authentication_error|invalid_api_key|permission_error|PERMISSION_DENIED|UNAUTHENTICATED/.test(hay)) return 'auth';
if (/insufficient_quota|billing_error/.test(hay)) return 'quota';
if (/context_length_exceeded|request_too_large/.test(hay)) return 'context_length';
if (/overloaded_error|server_error|api_error|UNAVAILABLE|INTERNAL\b/.test(`${f.type} ${f.code}`)) return 'server';
if (s === 401 || s === 403) return 'auth';
if (s === 402) return 'quota';
if (s === 429 || /rate_limit|RESOURCE_EXHAUSTED/.test(hay)) {
// Gemini words every 429 as "exceeded your current quota"; its details say
// whether it is a per-minute limit (retry soon) or a daily/billing cap.
if (/PerMinute|retryDelay|RetryInfo/.test(f.raw) && !/PerDay/.test(f.raw)) return 'rate_limit';
return QUOTA_RE.test(hay) || /PerDay/.test(f.raw) ? 'quota' : 'rate_limit';
}
if (s === 404 || /not_found_error|model_not_found|NOT_FOUND/.test(hay)) return 'not_found';
if (s === 408 || s === 504 || /DEADLINE_EXCEEDED/.test(hay)) return 'timeout';
if (s === 413) return 'context_length';
if (s >= 500) return 'server';
// Remaining 4xx (and status-less stream errors).
if (AUTH_RE.test(hay)) return 'auth'; // Gemini answers a bad key with 400 INVALID_ARGUMENT
if (CONTEXT_RE.test(hay)) return 'context_length';
if (QUOTA_RE.test(hay)) return 'quota'; // Anthropic: 400 "credit balance is too low"
if (FILTER_RE.test(hay)) return 'content_filter';
if (s === 0 && !f.message) return 'server';
return 'bad_request';
}
function friendlyMessage(code: AIErrorCode, provider: string): string {
switch (code) {
case 'auth':
return `${provider} rejected the API key.`;
case 'rate_limit':
return `${provider} is rate limiting requests. Try again in a moment.`;
case 'quota':
return `${provider} reports that the account is out of quota or credit.`;
case 'context_length':
return `The conversation is too long for this ${provider} model.`;
case 'not_found':
return `${provider} could not find that model or endpoint.`;
case 'content_filter':
return `${provider} blocked the request with its content filter.`;
case 'server':
return `${provider} is having trouble right now. Try again shortly.`;
case 'timeout':
return `${provider} took too long to respond.`;
case 'bad_request':
return `${provider} rejected the request.`;
default:
return `${provider} returned an error.`;
}
}
/**
* The single place that maps "HTTP status + provider error body" to an AIError.
* Pass `status: undefined` for errors that arrive as events inside a stream.
*/
export function errorFromProvider(ctx: ErrorContext, status: number | undefined, body: unknown): AIError {
const fields = extractProviderError(body);
const code = classify(status, fields);
const detail = fields.message || (status ? `HTTP ${status}` : '');
return new AIError(code, friendlyMessage(code, ctx.provider), {
status,
providerMessage: detail ? clip(scrubSecrets(detail, ctx.secrets)) : undefined,
});
}
/** Maps anything thrown by `fetch` / a body read to an AIError. */
export function errorFromThrown(ctx: ErrorContext, err: unknown, abortReason?: 'timeout' | 'aborted' | null): AIError {
if (err instanceof AIError) return err;
const name = isRecord(err) && typeof err.name === 'string' ? err.name : '';
if (abortReason === 'timeout' || name === 'TimeoutError') {
return new AIError('timeout', friendlyMessage('timeout', ctx.provider), { cause: err });
}
if (abortReason === 'aborted' || name === 'AbortError') {
return new AIError('aborted', 'The request was cancelled.', { cause: err });
}
// undici reports DNS/connection failures as TypeError("fetch failed") with the real reason in `cause`.
const cause = isRecord(err) && isRecord(err.cause) ? err.cause : undefined;
const parts = [
cause && typeof cause.code === 'string' ? cause.code : '',
cause && typeof cause.message === 'string' ? cause.message : '',
err instanceof Error ? err.message : String(err),
].filter(Boolean);
return new AIError('network', `Could not reach ${ctx.provider}. Check the address and your connection.`, {
providerMessage: clip(scrubSecrets(parts.join(': '), ctx.secrets)),
cause: err,
});
}
/* ------------------------------------------------------------------ */
/* fetch with timeouts */
/* ------------------------------------------------------------------ */
export interface HttpOptions extends ErrorContext {
method?: 'GET' | 'POST';
headers?: Record<string, string>;
/** Serialised as JSON when present. */
body?: unknown;
signal?: AbortSignal;
/** Time allowed until response headers arrive (default 60s). */
firstByteTimeoutMs?: number;
/** Hard cap on the whole exchange (default 30 min — streams are long). */
totalTimeoutMs?: number;
}
export interface HttpResponse {
response: Response;
ctx: ErrorContext;
/** Abort the exchange from our side (used by the idle watchdog). */
abort(reason: 'timeout' | 'aborted'): void;
abortReason(): 'timeout' | 'aborted' | null;
/** Clears timers/listeners. Safe to call more than once. */
cleanup(): void;
}
export const DEFAULT_FIRST_BYTE_TIMEOUT_MS = 60_000;
export const DEFAULT_TOTAL_TIMEOUT_MS = 30 * 60_000;
/** Longest silence tolerated inside an open stream before we give up. */
export const DEFAULT_IDLE_TIMEOUT_MS = 5 * 60_000;
/**
* Sends a request and resolves once headers are in. Does NOT check `response.ok`
* (callers decide; see `ensureOk`). Throws AIError for abort/timeout/network.
*/
export async function httpRequest(url: string, opts: HttpOptions): Promise<HttpResponse> {
const ctx: ErrorContext = { provider: opts.provider, secrets: opts.secrets };
if (opts.signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
const controller = new AbortController();
let reason: 'timeout' | 'aborted' | null = null;
const abort = (r: 'timeout' | 'aborted') => {
if (reason) return;
reason = r;
controller.abort();
};
const onUserAbort = () => abort('aborted');
opts.signal?.addEventListener('abort', onUserAbort, { once: true });
const firstByte = setTimeout(() => abort('timeout'), opts.firstByteTimeoutMs ?? DEFAULT_FIRST_BYTE_TIMEOUT_MS);
const total = setTimeout(() => abort('timeout'), opts.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS);
let cleaned = false;
const cleanup = () => {
if (cleaned) return;
cleaned = true;
clearTimeout(firstByte);
clearTimeout(total);
opts.signal?.removeEventListener('abort', onUserAbort);
};
const headers: Record<string, string> = { ...opts.headers };
let body: string | undefined;
if (opts.body !== undefined) {
body = JSON.stringify(opts.body);
if (!hasHeader(headers, 'content-type')) headers['Content-Type'] = 'application/json';
}
try {
const response = await fetch(url, {
method: opts.method ?? (body === undefined ? 'GET' : 'POST'),
headers,
body,
signal: controller.signal,
});
clearTimeout(firstByte);
return { response, ctx, abort, abortReason: () => reason, cleanup };
} catch (err) {
cleanup();
throw errorFromThrown(ctx, err, reason);
}
}
/** Reads the body of a failed response and throws the mapped AIError. Returns normally when `ok`. */
export async function ensureOk(http: HttpResponse): Promise<void> {
if (http.response.ok) return;
let text = '';
try {
text = await http.response.text();
} catch {
/* body unreadable — the status code alone still maps to something useful */
} finally {
http.cleanup();
}
throw errorFromProvider(http.ctx, http.response.status, text);
}
/** Request + status check + JSON parse, for non-streaming calls such as model discovery. */
export async function requestJson<T = unknown>(url: string, opts: HttpOptions): Promise<T> {
const http = await httpRequest(url, opts);
await ensureOk(http);
try {
const text = await http.response.text();
try {
return JSON.parse(text) as T;
} catch {
throw new AIError('invalid_response', `${opts.provider} sent a response that was not valid JSON.`, {
providerMessage: clip(scrubSecrets(text, opts.secrets), 300),
});
}
} catch (err) {
throw errorFromThrown(http.ctx, err, http.abortReason());
} finally {
http.cleanup();
}
}
/* ------------------------------------------------------------------ */
/* Server-sent events */
/* ------------------------------------------------------------------ */
export interface SSEMessage {
/** Value of the `event:` field, when the server sent one. */
event?: string;
/** All `data:` lines of the event joined with "\n". */
data: string;
}
export interface SSEOptions {
/** Abort when no bytes arrive for this long. */
idleTimeoutMs?: number;
onIdle?: () => void;
}
/**
* Parses a text/event-stream body. Handles `\n`, `\r\n` and bare `\r` line
* endings (also when split across chunks), multi-line `data:`, comment lines,
* a UTF-8 BOM, and a final event that is not followed by a blank line.
* Iteration ends at end-of-stream or at the OpenAI-style `data: [DONE]` sentinel.
*/
export async function* parseSSE(stream: ReadableStream<Uint8Array>, opts: SSEOptions = {}): AsyncGenerator<SSEMessage> {
const reader = stream.getReader();
const decoder = new TextDecoder('utf-8');
let buffer = '';
let dataLines: string[] = [];
let eventName: string | undefined;
let sawCR = false; // previous chunk ended in "\r": swallow a leading "\n"
let first = true;
const takeLine = (line: string): SSEMessage | 'done' | null => {
if (line === '') {
// Blank line dispatches the event.
if (dataLines.length === 0) {
eventName = undefined;
return null;
}
const msg: SSEMessage = { data: dataLines.join('\n') };
if (eventName !== undefined) msg.event = eventName;
dataLines = [];
eventName = undefined;
return msg.data === '[DONE]' ? 'done' : msg;
}
if (line.startsWith(':')) return null; // comment / keep-alive
const colon = line.indexOf(':');
const field = colon === -1 ? line : line.slice(0, colon);
let value = colon === -1 ? '' : line.slice(colon + 1);
if (value.startsWith(' ')) value = value.slice(1);
if (field === 'data') dataLines.push(value);
else if (field === 'event') eventName = value;
// `id` and `retry` are irrelevant for one-shot completions.
return null;
};
try {
for (;;) {
const { done, value } = await readWithIdle(reader, opts);
if (done) break;
let chunk = decoder.decode(value, { stream: true });
if (first && chunk.length > 0) {
first = false;
if (chunk.charCodeAt(0) === 0xfeff) chunk = chunk.slice(1);
}
if (sawCR && chunk.startsWith('\n')) chunk = chunk.slice(1);
if (chunk.length > 0) sawCR = chunk.endsWith('\r');
buffer += chunk;
let start = 0;
for (;;) {
const idx = indexOfLineEnd(buffer, start);
if (idx === -1) break;
const line = buffer.slice(start, idx);
start = buffer[idx] === '\r' && buffer[idx + 1] === '\n' ? idx + 2 : idx + 1;
const out = takeLine(line);
if (out === 'done') return;
if (out) yield out;
}
buffer = buffer.slice(start);
}
// Flush: tolerate a last line / last event without the terminating newline(s).
buffer += decoder.decode();
if (buffer.length > 0) {
const out = takeLine(buffer);
if (out === 'done') return;
if (out) yield out;
}
const tail = takeLine('');
if (tail && tail !== 'done') yield tail;
} finally {
try {
await reader.cancel();
} catch {
/* already closed or errored */
}
}
}
function indexOfLineEnd(s: string, from: number): number {
for (let i = from; i < s.length; i++) {
const c = s.charCodeAt(i);
if (c === 10 || c === 13) return i;
}
return -1;
}
async function readWithIdle(
reader: ReadableStreamDefaultReader<Uint8Array>,
opts: SSEOptions,
): Promise<Awaited<ReturnType<ReadableStreamDefaultReader<Uint8Array>['read']>>> {
if (!opts.idleTimeoutMs) return reader.read();
let timer: ReturnType<typeof setTimeout> | undefined;
const idle = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
opts.onIdle?.();
const err = new Error('stream idle timeout');
err.name = 'TimeoutError';
reject(err);
}, opts.idleTimeoutMs);
});
try {
return await Promise.race([reader.read(), idle]);
} finally {
clearTimeout(timer);
}
}
/**
* Iterates the SSE events of an open response, translating read failures
* (dropped connection, abort, idle timeout) into AIErrors and always releasing
* the request's timers. `data` is JSON-parsed; unparseable events are skipped.
*/
export async function* sseJson(http: HttpResponse, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS): AsyncGenerator<{ event?: string; data: unknown }> {
const body = http.response.body;
if (!body) {
http.cleanup();
throw new AIError('invalid_response', `${http.ctx.provider} returned an empty response.`);
}
const events = parseSSE(body, { idleTimeoutMs, onIdle: () => http.abort('timeout') });
try {
for (;;) {
let next: IteratorResult<SSEMessage>;
try {
next = await events.next();
} catch (err) {
throw errorFromThrown(http.ctx, err, http.abortReason());
}
if (next.done) return;
let data: unknown;
try {
data = JSON.parse(next.value.data);
} catch {
continue; // keep-alives such as OpenRouter's ": OPENROUTER PROCESSING" variants, or junk
}
yield next.value.event !== undefined ? { event: next.value.event, data } : { data };
}
} finally {
await events.return(undefined).catch(() => undefined);
http.cleanup();
}
}
/* ------------------------------------------------------------------ */
/* Small shared utilities */
/* ------------------------------------------------------------------ */
export function isRecord(v: unknown): v is Record<string, unknown> {
return typeof v === 'object' && v !== null && !Array.isArray(v);
}
function hasHeader(headers: Record<string, string>, name: string): boolean {
const n = name.toLowerCase();
return Object.keys(headers).some((k) => k.toLowerCase() === n);
}
function safeStringify(v: unknown): string {
try {
return JSON.stringify(v) ?? '';
} catch {
return '';
}
}
/** Trims whitespace and trailing slashes from a base URL. */
export function trimBaseUrl(url: string): string {
return url.trim().replace(/\/+$/, '');
}
/** Host part of a URL for user-facing messages; never includes credentials or query. */
export function hostOf(url: string): string {
try {
return new URL(url).host;
} catch {
return url;
}
}
/** True for loopback / private-network hosts, where a local model server may need minutes to load a model. */
export function isLocalUrl(url: string): boolean {
try {
const h = new URL(url).hostname.replace(/^\[|\]$/g, '');
return (
h === 'localhost' ||
h === '::1' ||
h.endsWith('.local') ||
h.endsWith('.internal') ||
/^127\./.test(h) ||
/^10\./.test(h) ||
/^192\.168\./.test(h) ||
/^172\.(1[6-9]|2\d|3[01])\./.test(h) ||
h === 'host.docker.internal'
);
} catch {
return false;
}
}

View file

@ -0,0 +1,26 @@
import { AIError, type ProviderAdapter, type ProviderKind } from '../types.ts';
import { anthropicAdapter } from './anthropic.ts';
import { geminiAdapter } from './gemini.ts';
import { ollamaAdapter } from './ollama.ts';
import { openaiAdapter } from './openai.ts';
import { openaiCompatibleAdapter } from './openaiCompatible.ts';
const ADAPTERS: Record<ProviderKind, ProviderAdapter> = {
openai: openaiAdapter,
anthropic: anthropicAdapter,
gemini: geminiAdapter,
openai_compatible: openaiCompatibleAdapter,
ollama: ollamaAdapter,
};
export function getAdapter(kind: ProviderKind): ProviderAdapter {
const adapter = ADAPTERS[kind];
if (!adapter) throw new AIError('unsupported', `Unknown provider type "${kind}".`);
return adapter;
}
export function listAdapterMeta(): Array<{ kind: ProviderKind } & ProviderAdapter['meta']> {
return (Object.keys(ADAPTERS) as ProviderKind[]).map((kind) => ({ kind, ...ADAPTERS[kind].meta }));
}
export { CATALOG_UPDATED, lookupModelDefaults } from './catalog.ts';

View file

@ -0,0 +1,102 @@
/**
* Ollama (local models). Chat goes through Ollama's OpenAI-compatible endpoint
* (`{base}/v1/chat/completions`), which handles tools and streaming; discovery
* uses the native API because it reports what each model can do:
* GET /api/tags -> installed models
* POST /api/show -> { capabilities: ["completion","tools","vision","thinking"], model_info }
* https://docs.ollama.com/api · https://docs.ollama.com/openai
*/
import { AIError, type DiscoveredModel, type ModelCapabilities, type ProviderAdapter, type ProviderConfig, type TestResult } from '../types.ts';
import { num, runTest, str, TEST_TIMEOUT_MS } from './common.ts';
import { isRecord, requestJson, trimBaseUrl, type ErrorContext } from './http.ts';
import { streamChatCompletions, type CompatOptions } from './openaiCompatible.ts';
const LABEL = 'Ollama';
const DEFAULT_BASE_URL = 'http://localhost:11434';
function root(cfg: ProviderConfig): string {
return trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL).replace(/\/(v1|api)$/, '');
}
function ctx(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
const h: Record<string, string> = { ...cfg.headers };
if (cfg.apiKey) h.Authorization = `Bearer ${cfg.apiKey}`; // only needed behind an authenticating proxy
return h;
}
function compat(cfg: ProviderConfig): CompatOptions {
return { kind: 'ollama', label: LABEL, baseUrl: `${root(cfg)}/v1`, apiKey: cfg.apiKey, headers: cfg.headers };
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
let tags: unknown;
try {
tags = await requestJson(`${root(cfg)}/api/tags`, { ...ctx(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs ?? 15_000, totalTimeoutMs: timeoutMs ?? 30_000 });
} catch (err) {
if (err instanceof AIError && err.code === 'network') {
throw new AIError('network', `Could not reach Ollama at ${root(cfg)}. Is it running, and reachable from the Study Desk container?`, { retryable: true });
}
throw err;
}
const list = isRecord(tags) && Array.isArray(tags.models) ? tags.models.filter(isRecord) : [];
const models: DiscoveredModel[] = [];
// /api/show is one cheap local call per model; cap it so a huge library cannot stall the settings page.
await Promise.all(
list.slice(0, 60).map(async (m) => {
const id = str(m.model) || str(m.name);
if (!id) return;
const model: DiscoveredModel = { id, label: str(m.name) || id, priceIn: 0, priceOut: 0 };
try {
const show = await requestJson<Record<string, unknown>>(`${root(cfg)}/api/show`, {
...ctx(cfg),
headers: headers(cfg),
body: { model: id },
signal,
firstByteTimeoutMs: 10_000,
totalTimeoutMs: 10_000,
});
if (Array.isArray(show.capabilities)) {
const caps = show.capabilities.map(String);
if (!caps.includes('completion')) return; // embedding-only model
const c: Partial<ModelCapabilities> = { tools: caps.includes('tools'), vision: caps.includes('vision'), reasoning: caps.includes('thinking'), pdf: false, temperature: true };
model.capabilities = c;
}
if (isRecord(show.model_info)) {
const key = Object.keys(show.model_info).find((k) => k.endsWith('.context_length'));
if (key && num(show.model_info[key])) model.contextWindow = num(show.model_info[key]);
}
} catch {
/* older Ollama versions: keep the model with unknown capabilities */
}
models.push(model);
}),
);
return models.sort((a, b) => a.id.localeCompare(b.id));
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => {
const count = (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length;
return count === 0 ? { modelCount: 0, note: 'Connected to Ollama, but no models are installed. Run `ollama pull <model>` first.' } : { modelCount: count };
});
}
export const ollamaAdapter: ProviderAdapter = {
kind: 'ollama',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: false,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://docs.ollama.com',
keyHint: 'Not needed',
},
stream: (cfg, req) => streamChatCompletions(compat(cfg), req),
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,313 @@
/**
* OpenAI — Responses API (`POST /v1/responses`, SSE).
* https://platform.openai.com/docs/api-reference/responses
* https://platform.openai.com/docs/guides/function-calling
* https://platform.openai.com/docs/guides/reasoning
*
* Quirks handled here:
* - Requests are stateless (`store: false`). Reasoning models need their reasoning
* items passed back between tool calls, so encrypted reasoning is requested via
* `include: ["reasoning.encrypted_content"]` and every turn's raw output items are
* kept in `providerData` and replayed verbatim.
* - Reasoning models reject `temperature`; classic models reject `reasoning`.
* - The accepted `reasoning.effort` values differ by model generation. When a model
* rejects the chosen value the request is retried once with a safe one.
* - Tool-call arguments arrive as deltas keyed by output item.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { lookupModelDefaults } from './catalog.ts';
import { dataUrl, isChatModelId, makeUsage, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, httpRequest, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
const LABEL = 'OpenAI';
const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
function base(cfg: ProviderConfig): string {
const url = trimBaseUrl(cfg.baseUrl || DEFAULT_BASE_URL);
return /\/v\d+$/.test(url) ? url : `${url}/v1`;
}
function errorContext(cfg: ProviderConfig): ErrorContext {
return { provider: LABEL, secrets: [cfg.apiKey, ...Object.values(cfg.headers ?? {})] };
}
function headers(cfg: ProviderConfig): Record<string, string> {
if (!cfg.apiKey) throw new AIError('not_configured', 'Add an OpenAI API key in Settings to use OpenAI models.');
return { ...cfg.headers, Authorization: `Bearer ${cfg.apiKey}` };
}
/** Whether a model is a reasoning model, when the caller did not say. */
function isReasoningModel(req: ChatRequest): boolean {
if (req.capabilities?.reasoning !== undefined) return req.capabilities.reasoning;
const known = lookupModelDefaults('openai', req.model).capabilities?.reasoning;
if (known !== undefined) return known;
return /^(o\d|gpt-[5-9]|gpt-\d{2})/.test(req.model) && !/chat/.test(req.model);
}
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): Json[] {
const out: Json[] = [];
for (const p of parts) {
if (p.type === 'text' && p.text) out.push({ type: 'input_text', text: p.text });
else if (p.type === 'image') {
if (caps.vision === false) out.push({ type: 'input_text', text: '[An image was attached, but this model cannot read images.]' });
else out.push({ type: 'input_image', image_url: dataUrl(p.mime, p.data) });
} else if (p.type === 'file') {
if (p.mime === 'application/pdf' && caps.pdf !== false) out.push({ type: 'input_file', filename: p.filename ?? 'document.pdf', file_data: dataUrl(p.mime, p.data) });
else if (p.mime.startsWith('text/')) out.push({ type: 'input_text', text: Buffer.from(p.data, 'base64').toString('utf8') });
else out.push({ type: 'input_text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
return out;
}
function toInput(messages: ChatMessage[], caps: Partial<ModelCapabilities>, dropReasoning: boolean): Json[] {
const input: Json[] = [];
for (const msg of messages) {
if (msg.role === 'assistant') {
const pd = msg.providerData;
if (pd?.kind === 'openai' && isRecord(pd.data) && Array.isArray(pd.data.output)) {
// Same provider: replay the output items exactly (reasoning items included).
for (const item of pd.data.output.filter(isRecord)) {
if (item.type === 'reasoning' && (dropReasoning || !item.encrypted_content)) continue;
input.push(item);
}
continue;
}
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
if (text) input.push({ role: 'assistant', content: [{ type: 'output_text', text }] });
for (const p of msg.content) {
if (p.type === 'tool_call') input.push({ type: 'function_call', call_id: p.id, name: p.name, arguments: JSON.stringify(p.input ?? {}) });
}
continue;
}
for (const p of msg.content) {
if (p.type === 'tool_result') input.push({ type: 'function_call_output', call_id: p.toolCallId, output: p.output || '(no output)' });
}
const content = userContent(msg.content.filter((p) => p.type !== 'tool_result'), caps);
if (content.length > 0) input.push({ role: 'user', content });
}
return input;
}
interface Flags {
/** Value sent as reasoning.effort, or null to omit the field. */
effort: string | null;
summary: boolean;
temperature: boolean;
dropReasoning: boolean;
include: boolean;
}
function buildBody(req: ChatRequest, flags: Flags): Json {
const caps = req.capabilities ?? {};
const body: Json = { model: req.model, input: toInput(req.messages, caps, flags.dropReasoning), stream: true, store: false };
if (req.system?.trim()) body.instructions = req.system.trim();
if (req.maxOutputTokens) body.max_output_tokens = Math.max(16, req.maxOutputTokens);
if (caps.tools !== false && req.tools?.length) {
body.tools = req.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters }, strict: false }));
const choice = req.toolChoice;
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
else if (isRecord(choice)) body.tool_choice = { type: 'function', name: choice.name };
}
if (isReasoningModel(req)) {
const reasoning: Json = {};
if (flags.effort) reasoning.effort = flags.effort;
if (flags.summary) reasoning.summary = 'auto';
if (Object.keys(reasoning).length > 0) body.reasoning = reasoning;
if (flags.include) body.include = ['reasoning.encrypted_content'];
} else if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) {
body.temperature = req.temperature;
}
if (req.jsonMode) body.text = { format: { type: 'json_object' } };
return body;
}
async function* stream(cfg: ProviderConfig, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(cfg);
const flags: Flags = {
// "none" is only understood by recent models; if it is rejected we fall back to "low".
effort: req.reasoningEffort ?? null,
summary: true,
temperature: true,
dropReasoning: false,
include: true,
};
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(`${base(cfg)}/responses`, { ...ctx, headers: headers(cfg), body: buildBody(req, flags), signal: req.signal });
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || err.status !== 400 || attempt >= 5) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (flags.effort && /reasoning\.effort|reasoning_effort|effort/.test(detail)) flags.effort = flags.effort === 'none' ? 'minimal' : flags.effort === 'minimal' ? 'low' : null;
else if (flags.summary && /summary/.test(detail)) flags.summary = false;
else if (flags.include && /include|encrypted_content/.test(detail)) flags.include = false;
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
else if (!flags.dropReasoning && /reasoning/.test(detail) && /item|input/.test(detail)) flags.dropReasoning = true;
else throw err;
}
}
const args = new Map<string, string>();
let final: Json | null = null;
let reasoningEmitted = false;
let streamedText = '';
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
switch (data.type) {
case 'response.output_text.delta': {
const delta = str(data.delta);
if (delta) {
streamedText += delta;
yield { type: 'text', delta };
}
break;
}
case 'response.refusal.delta': {
const delta = str(data.delta);
if (delta) yield { type: 'text', delta };
break;
}
case 'response.reasoning_summary_part.added':
if (reasoningEmitted) yield { type: 'reasoning', delta: '\n\n' };
break;
case 'response.reasoning_summary_text.delta':
case 'response.reasoning_text.delta': {
const delta = str(data.delta);
if (delta) {
reasoningEmitted = true;
yield { type: 'reasoning', delta };
}
break;
}
case 'response.function_call_arguments.delta':
args.set(str(data.item_id), (args.get(str(data.item_id)) ?? '') + str(data.delta));
break;
case 'response.completed':
case 'response.incomplete':
if (isRecord(data.response)) final = data.response;
break;
case 'response.failed':
throw errorFromProvider(ctx, undefined, isRecord(data.response) && isRecord(data.response.error) ? { error: data.response.error } : data);
case 'error':
throw errorFromProvider(ctx, undefined, isRecord(data.error) ? data : { error: data });
default:
break;
}
}
if (!final) throw new AIError('network', 'The connection to OpenAI closed before the response finished.', { retryable: true });
const incomplete = isRecord(final.incomplete_details) ? str(final.incomplete_details.reason) : '';
const truncated = incomplete === 'max_output_tokens';
const output = Array.isArray(final.output) ? final.output.filter(isRecord) : [];
const content: ContentPart[] = [];
const replay: Json[] = [];
const calls: { id: string; name: string; input: Record<string, unknown> }[] = [];
let refused = false;
let text = '';
for (const item of output) {
if (item.type === 'message') {
for (const part of Array.isArray(item.content) ? item.content.filter(isRecord) : []) {
if (part.type === 'output_text') text += str(part.text);
else if (part.type === 'refusal') {
refused = true;
text += str(part.refusal);
}
}
replay.push(item);
} else if (item.type === 'function_call') {
const name = str(item.name);
const raw = str(item.arguments) || args.get(str(item.id)) || '';
let input = parseArgsStrict(raw);
if (!input) {
if (truncated) continue;
input = parseToolArguments(raw, LABEL, name);
}
calls.push({ id: str(item.call_id) || str(item.id), name, input });
replay.push(item);
} else if (item.type === 'reasoning') {
replay.push(item);
}
}
if (!text && streamedText) text = streamedText;
if (text) content.push({ type: 'text', text });
for (const call of calls) {
content.push({ type: 'tool_call', ...call });
yield { type: 'tool_call', call };
}
let stopReason: StopReason = 'end';
if (calls.length > 0) stopReason = 'tool_use';
else if (refused || incomplete === 'content_filter') stopReason = 'refusal';
else if (truncated) stopReason = 'max_tokens';
else if (incomplete) stopReason = 'other';
const u = isRecord(final.usage) ? final.usage : {};
const message: ChatMessage = { role: 'assistant', content };
if (replay.length > 0) message.providerData = { kind: 'openai', data: { output: replay } };
yield {
type: 'done',
message,
stopReason,
usage: makeUsage({
input: num(u.input_tokens),
output: num(u.output_tokens),
cacheRead: isRecord(u.input_tokens_details) ? num(u.input_tokens_details.cached_tokens) : 0,
reasoning: isRecord(u.output_tokens_details) ? num(u.output_tokens_details.reasoning_tokens) : 0,
}),
};
}
async function listModels(cfg: ProviderConfig, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const res = await requestJson<Json>(`${base(cfg)}/models`, { ...errorContext(cfg), headers: headers(cfg), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
const data = Array.isArray(res.data) ? res.data.filter(isRecord) : [];
const found: DiscoveredModel[] = [];
for (const m of data) {
const id = str(m.id);
// Fine-tunes ("ft:...") are chat models too; search/codex-only variants are not useful here.
if (!id || !isChatModelId(id) || /search|^codex-|-codex-max/.test(id)) continue;
if (!/^(gpt|o\d|chat|ft:)/.test(id)) continue;
found.push({ id });
}
found.sort((a, b) => b.id.localeCompare(a.id, undefined, { numeric: true }));
return withCatalogDefaults('openai', found);
}
async function test(cfg: ProviderConfig): Promise<TestResult> {
return runTest(LABEL, async () => ({ modelCount: (await listModels(cfg, undefined, TEST_TIMEOUT_MS)).length }));
}
export const openaiAdapter: ProviderAdapter = {
kind: 'openai',
meta: {
label: LABEL,
defaultBaseUrl: DEFAULT_BASE_URL,
requiresApiKey: true,
requiresBaseUrl: false,
supportsDiscovery: true,
docsUrl: 'https://platform.openai.com/api-keys',
keyHint: 'sk-…',
},
stream,
listModels: (cfg, signal) => listModels(cfg, signal),
test,
};

View file

@ -0,0 +1,363 @@
/**
* Generic Chat Completions (`POST {base}/chat/completions`) for OpenRouter, Groq,
* Together, LM Studio, vLLM, llama.cpp and friends, and the engine behind the
* Ollama adapter.
*
* These servers agree on the basics and disagree on everything else, so this
* adapter is deliberately forgiving:
* - `stream_options.include_usage`, `tools`, `response_format`, `max_tokens` vs
* `max_completion_tokens` and `reasoning_effort` are each dropped and retried
* once when a server answers 400 about them;
* - usage may be missing, and may arrive on a chunk with no choices;
* - tool-call argument fragments may arrive in any chunking, with or without ids;
* - reasoning text may arrive as `reasoning_content`, `reasoning` or `thinking`;
* - the last chunk may lack a `finish_reason`.
*/
import {
AIError,
type ChatMessage,
type ChatRequest,
type ContentPart,
type DiscoveredModel,
type ModelCapabilities,
type ProviderAdapter,
type ProviderConfig,
type ProviderKind,
type StopReason,
type StreamEvent,
type TestResult,
} from '../types.ts';
import { dataUrl, isChatModelId, makeUsage, newToolCallId, num, parseArgsStrict, parseToolArguments, runTest, str, TEST_TIMEOUT_MS, withCatalogDefaults } from './common.ts';
import { ensureOk, errorFromProvider, hostOf, httpRequest, isLocalUrl, isRecord, requestJson, sseJson, trimBaseUrl, type ErrorContext, type HttpResponse } from './http.ts';
type Json = Record<string, unknown>;
/** Accepts ".../v1", ".../v1/", ".../v1/chat/completions" or a bare host and returns the API root. */
export function normalizeCompatBase(url: string): string {
let base = trimBaseUrl(url).replace(/\/(chat\/completions|completions|models)$/, '');
try {
const u = new URL(base);
// A bare host ("http://localhost:1234") almost always means the conventional /v1 root.
if (u.pathname === '' || u.pathname === '/') base = `${trimBaseUrl(base)}/v1`;
} catch {
/* validated by the caller */
}
return base;
}
export interface CompatOptions {
kind: ProviderKind;
label: string;
baseUrl: string;
apiKey: string | null;
headers?: Record<string, string>;
}
function userContent(parts: ContentPart[], caps: Partial<ModelCapabilities>): string | Json[] {
const blocks: Json[] = [];
for (const p of parts) {
if (p.type === 'text' && p.text) blocks.push({ type: 'text', text: p.text });
else if (p.type === 'image') {
if (caps.vision === false) blocks.push({ type: 'text', text: '[An image was attached, but this model cannot read images.]' });
else blocks.push({ type: 'image_url', image_url: { url: dataUrl(p.mime, p.data) } });
} else if (p.type === 'file') {
if (p.mime.startsWith('text/')) blocks.push({ type: 'text', text: Buffer.from(p.data, 'base64').toString('utf8') });
else blocks.push({ type: 'text', text: `[The file "${p.filename ?? 'attachment'}" was attached, but this model cannot read it directly.]` });
}
}
// Plain strings are understood by every server; arrays only by multimodal ones.
if (blocks.every((b) => b.type === 'text')) return blocks.map((b) => str(b.text)).join('\n\n');
return blocks;
}
export function toChatMessages(req: ChatRequest, systemRole: 'system' | 'developer' = 'system'): Json[] {
const caps = req.capabilities ?? {};
const out: Json[] = [];
if (req.system?.trim()) out.push({ role: systemRole, content: req.system.trim() });
for (const msg of req.messages) {
if (msg.role === 'assistant') {
const text = msg.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
const calls = msg.content.filter((p) => p.type === 'tool_call') as Extract<ContentPart, { type: 'tool_call' }>[];
if (!text && calls.length === 0) continue;
const wire: Json = { role: 'assistant', content: text || null };
if (calls.length > 0) {
wire.tool_calls = calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: JSON.stringify(c.input ?? {}) } }));
}
out.push(wire);
continue;
}
// Tool results must directly follow the assistant turn that requested them.
for (const p of msg.content) {
if (p.type === 'tool_result') out.push({ role: 'tool', tool_call_id: p.toolCallId, content: p.output || '(no output)' });
}
const rest = msg.content.filter((p) => p.type !== 'tool_result' && p.type !== 'tool_call');
if (rest.length > 0) {
const content = userContent(rest, caps);
if (typeof content !== 'string' || content) out.push({ role: 'user', content });
}
}
return out;
}
interface BodyFlags {
tools: boolean;
usage: boolean;
jsonMode: boolean;
temperature: boolean;
reasoning: boolean;
maxKey: 'max_tokens' | 'max_completion_tokens' | null;
}
function buildBody(req: ChatRequest, flags: BodyFlags): Json {
const caps = req.capabilities ?? {};
const body: Json = { model: req.model, messages: toChatMessages(req), stream: true };
if (flags.usage) body.stream_options = { include_usage: true };
if (flags.tools && caps.tools !== false && req.tools?.length) {
body.tools = req.tools.map((t) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: { type: 'object', ...t.parameters } } }));
const choice = req.toolChoice;
if (choice === 'none' || choice === 'required' || choice === 'auto') body.tool_choice = choice;
else if (isRecord(choice)) body.tool_choice = { type: 'function', function: { name: choice.name } };
}
if (flags.temperature && req.temperature !== undefined && caps.temperature !== false) body.temperature = req.temperature;
if (flags.maxKey && req.maxOutputTokens) body[flags.maxKey] = req.maxOutputTokens;
if (flags.reasoning && caps.reasoning === true && req.reasoningEffort && req.reasoningEffort !== 'none') body.reasoning_effort = req.reasoningEffort;
if (flags.jsonMode && req.jsonMode) body.response_format = { type: 'json_object' };
return body;
}
function mapFinish(reason: string): StopReason {
switch (reason) {
case 'stop':
case 'eos':
case 'end_turn':
return 'end';
case 'tool_calls':
case 'function_call':
return 'tool_use';
case 'length':
case 'max_tokens':
return 'max_tokens';
case 'content_filter':
return 'refusal';
default:
return 'other';
}
}
function authHeaders(o: CompatOptions): Record<string, string> {
const h: Record<string, string> = { ...o.headers };
if (o.apiKey) h.Authorization = `Bearer ${o.apiKey}`;
return h;
}
function errorContext(o: CompatOptions): ErrorContext {
return { provider: o.label, secrets: [o.apiKey, ...Object.values(o.headers ?? {})] };
}
export async function* streamChatCompletions(o: CompatOptions, req: ChatRequest): AsyncGenerator<StreamEvent> {
const ctx = errorContext(o);
const base = normalizeCompatBase(o.baseUrl);
const local = isLocalUrl(base);
const flags: BodyFlags = { tools: true, usage: true, jsonMode: true, temperature: true, reasoning: true, maxKey: 'max_tokens' };
let http: HttpResponse;
for (let attempt = 0; ; attempt += 1) {
http = await httpRequest(`${base}/chat/completions`, {
...ctx,
headers: authHeaders(o),
body: buildBody(req, flags),
signal: req.signal,
// A local server may spend minutes loading a model into memory before the first byte.
firstByteTimeoutMs: local ? 10 * 60_000 : undefined,
});
try {
await ensureOk(http);
break;
} catch (err) {
if (!(err instanceof AIError) || (err.status !== 400 && err.status !== 422) || attempt >= 6) throw err;
const detail = (err.providerMessage ?? '').toLowerCase();
if (flags.usage && /stream_options|include_usage/.test(detail)) flags.usage = false;
else if (flags.maxKey === 'max_tokens' && /max_completion_tokens/.test(detail)) flags.maxKey = 'max_completion_tokens';
else if (flags.maxKey && /max_tokens|max_completion_tokens/.test(detail)) flags.maxKey = null;
else if (flags.reasoning && /reasoning_effort|reasoning/.test(detail) && req.reasoningEffort) flags.reasoning = false;
else if (flags.temperature && /temperature/.test(detail)) flags.temperature = false;
else if (flags.jsonMode && req.jsonMode && /response_format|json_object|json mode/.test(detail)) flags.jsonMode = false;
else if (flags.tools && req.tools?.length && /tool|function/.test(detail)) {
// The model (or server) cannot do tool calling at all.
if (req.toolChoice === 'required' || isRecord(req.toolChoice)) {
throw new AIError('unsupported', `${o.label} reports that "${req.model}" does not support tool calling. Pick a different model for this task.`, {
status: err.status,
providerMessage: err.providerMessage,
});
}
flags.tools = false;
} else throw err;
}
}
let text = '';
let finish = '';
let reasoningSeen = false;
const usage = { input: 0, output: 0, cacheRead: 0, reasoning: 0 };
const calls: { id: string; name: string; args: string }[] = [];
for await (const { data } of sseJson(http)) {
if (!isRecord(data)) continue;
if (isRecord(data.error)) throw errorFromProvider(ctx, undefined, data);
if (isRecord(data.usage)) {
const u = data.usage;
usage.input = num(u.prompt_tokens) || usage.input;
usage.output = num(u.completion_tokens) || usage.output;
if (isRecord(u.prompt_tokens_details)) usage.cacheRead = num(u.prompt_tokens_details.cached_tokens);
if (isRecord(u.completion_tokens_details)) usage.reasoning = num(u.completion_tokens_details.reasoning_tokens);
}
const choice = Array.isArray(data.choices) ? data.choices[0] : undefined;
if (!isRecord(choice)) continue;
if (typeof choice.finish_reason === 'string' && choice.finish_reason) finish = choice.finish_reason;
const delta = isRecord(choice.delta) ? choice.delta : isRecord(choice.message) ? choice.message : null;
if (!delta) continue;
const reasoning = str(delta.reasoning_content) || str(delta.reasoning) || str(delta.thinking);
if (reasoning) {
reasoningSeen = true;
yield { type: 'reasoning', delta: reasoning };
}
const piece = typeof delta.content === 'string' ? delta.content : '';
if (piece) {
text += piece;
yield { type: 'text', delta: piece };
}
if (Array.isArray(delta.tool_calls)) {
for (const raw of delta.tool_calls) {
if (!isRecord(raw)) continue;
const fn = isRecord(raw.function) ? raw.function : {};
// `index` identifies the call; servers that omit it send one whole call per entry.
const index = typeof raw.index === 'number' ? raw.index : str(raw.id) ? calls.findIndex((c) => c.id === raw.id) : calls.length - 1;
let slot = index >= 0 ? calls[index] : undefined;
if (!slot) {
slot = { id: '', name: '', args: '' };
if (index >= 0) calls[index] = slot;
else calls.push(slot);
}
if (str(raw.id)) slot.id = str(raw.id);
if (str(fn.name)) slot.name = slot.name ? slot.name : str(fn.name);
if (typeof fn.arguments === 'string') slot.args += fn.arguments;
else if (isRecord(fn.arguments)) slot.args = JSON.stringify(fn.arguments);
}
}
}
const stop = finish ? mapFinish(finish) : 'end';
const content: ContentPart[] = [];
if (text) content.push({ type: 'text', text });
const done: { id: string; name: string; input: Record<string, unknown> }[] = [];
for (const c of calls) {
if (!c || !c.name) continue;
let input = parseArgsStrict(c.args);
if (!input) {
if (stop === 'max_tokens') continue; // cut off mid-arguments: never guess the rest
input = parseToolArguments(c.args, o.label, c.name);
}
done.push({ id: c.id || newToolCallId('call'), name: c.name, input });
}
for (const call of done) {
content.push({ type: 'tool_call', ...call });
yield { type: 'tool_call', call };
}
if (content.length === 0 && !reasoningSeen && !finish) {
throw new AIError('invalid_response', `${o.label} returned an empty response.`, { retryable: true });
}
const message: ChatMessage = { role: 'assistant', content };
yield {
type: 'done',
message,
stopReason: done.length > 0 ? 'tool_use' : stop === 'tool_use' ? 'end' : stop,
usage: makeUsage({ input: usage.input, output: usage.output, cacheRead: usage.cacheRead, reasoning: usage.reasoning }),
};
}
export async function listCompatModels(o: CompatOptions, signal?: AbortSignal, timeoutMs?: number): Promise<DiscoveredModel[]> {
const base = normalizeCompatBase(o.baseUrl);
let res: unknown;
try {
res = await requestJson(`${base}/models`, { ...errorContext(o), headers: authHeaders(o), signal, firstByteTimeoutMs: timeoutMs, totalTimeoutMs: timeoutMs });
} catch (err) {
if (err instanceof AIError && (err.code === 'not_found' || err.code === 'bad_request')) {
throw new AIError('unsupported', `${hostOf(base)} does not list its models. Add model ids manually.`, { status: err.status });
}
throw err;
}
const list = isRecord(res) && Array.isArray(res.data) ? res.data : Array.isArray(res) ? res : isRecord(res) && Array.isArray(res.models) ? res.models : [];
const found: DiscoveredModel[] = [];
for (const m of list) {
if (!isRecord(m)) continue;
const id = str(m.id) || str(m.name);
if (!id || !isChatModelId(id)) continue;
const model: DiscoveredModel = { id };
if (str(m.name) && str(m.name) !== id) model.label = str(m.name);
// OpenRouter-style metadata, when present.
const ctxLen = num(m.context_length) || num(m.context_window) || num(m.max_context_length);
if (ctxLen) model.contextWindow = ctxLen;
if (isRecord(m.top_provider) && num(m.top_provider.max_completion_tokens)) model.maxOutput = num(m.top_provider.max_completion_tokens);
if (isRecord(m.pricing)) {
const pin = Number(m.pricing.prompt);
const pout = Number(m.pricing.completion);
if (Number.isFinite(pin) && pin >= 0) model.priceIn = Math.round(pin * 1e9) / 1e3;
if (Number.isFinite(pout) && pout >= 0) model.priceOut = Math.round(pout * 1e9) / 1e3;
}
const caps: Partial<ModelCapabilities> = {};
if (Array.isArray(m.supported_parameters)) {
caps.tools = m.supported_parameters.includes('tools');
caps.reasoning = m.supported_parameters.includes('reasoning') || m.supported_parameters.includes('reasoning_effort');
}
if (isRecord(m.architecture) && Array.isArray(m.architecture.input_modalities)) {
caps.vision = m.architecture.input_modalities.includes('image');
}
if (Object.keys(caps).length > 0) model.capabilities = caps;
found.push(model);
}
return withCatalogDefaults(o.kind, found);
}
function options(cfg: ProviderConfig): CompatOptions {
if (!cfg.baseUrl?.trim()) throw new AIError('not_configured', `Add the server URL for "${cfg.name}" in Settings.`);
return { kind: 'openai_compatible', label: cfg.name || hostOf(cfg.baseUrl), baseUrl: cfg.baseUrl, apiKey: cfg.apiKey, headers: cfg.headers };
}
async function test(cfg: ProviderConfig, model?: string): Promise<TestResult> {
return runTest(cfg.name || 'the server', async () => {
const o = options(cfg);
try {
return { modelCount: (await listCompatModels(o, undefined, TEST_TIMEOUT_MS)).length };
} catch (err) {
if (!(err instanceof AIError) || err.code !== 'unsupported' || !model) throw err;
// No model listing: prove the connection with a one-token completion instead.
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS);
try {
for await (const ev of streamChatCompletions(o, { model, messages: [{ role: 'user', content: [{ type: 'text', text: 'Hi' }] }], maxOutputTokens: 1, signal: controller.signal })) {
if (ev.type === 'done') break;
}
} finally {
clearTimeout(timer);
}
return { note: `Connected to ${o.label}. This server does not list models, so add model ids manually.` };
}
});
}
export const openaiCompatibleAdapter: ProviderAdapter = {
kind: 'openai_compatible',
meta: {
label: 'OpenAI-compatible',
defaultBaseUrl: null,
requiresApiKey: false,
requiresBaseUrl: true,
supportsDiscovery: true,
docsUrl: 'https://openrouter.ai/docs/quickstart',
keyHint: 'Optional for local servers',
},
stream: (cfg, req) => streamChatCompletions(options(cfg), req),
listModels: (cfg, signal) => listCompatModels(options(cfg), signal),
test,
};

View file

@ -0,0 +1,932 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
/**
* Provider adapter tests. `fetch` is always stubbed; no network is touched.
* Wire shapes below are derived from what each adapter's parser reads.
*/
import { afterEach, describe, expect, it, vi } from 'vitest';
import { AIError, type ChatMessage, type ChatRequest, type ProviderAdapter, type ProviderConfig, type ProviderKind, type StreamEvent, type ToolDef, type Usage } from '../types.ts';
import { anthropicAdapter } from './anthropic.ts';
import { lookupModelDefaults } from './catalog.ts';
import { toGeminiSchema, geminiAdapter } from './gemini.ts';
import { parseSSE } from './http.ts';
import { ollamaAdapter } from './ollama.ts';
import { openaiAdapter } from './openai.ts';
import { normalizeCompatBase, openaiCompatibleAdapter } from './openaiCompatible.ts';
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
const enc = new TextEncoder();
function bytesStream(chunks: Uint8Array[]): ReadableStream<Uint8Array> {
return new ReadableStream({
start(c) {
for (const ch of chunks) c.enqueue(ch);
c.close();
},
});
}
const textStream = (chunks: string[]) => bytesStream(chunks.map((c) => enc.encode(c)));
/** One SSE event. */
function ev(data: unknown, event?: string): string {
return `${event ? `event: ${event}\n` : ''}data: ${JSON.stringify(data)}\n\n`;
}
const DONE = 'data: [DONE]\n\n';
function rechunk(s: string, n: number): string[] {
const out: string[] = [];
for (let i = 0; i < s.length; i += n) out.push(s.slice(i, i + n));
return out;
}
/** SSE response whose chunk boundaries deliberately fall mid-line. */
function sseResponse(chunks: string[], status = 200): Response {
return new Response(textStream(rechunk(chunks.join(''), 23)), { status, headers: { 'content-type': 'text/event-stream' } });
}
function jsonResponse(body: unknown, status = 200): Response {
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
}
interface Call {
url: string;
method?: string;
headers: Record<string, string>;
body: any;
}
let calls: Call[] = [];
type Reply = Response | Error;
type Handler = (call: Call, n: number) => Reply | Promise<Reply>;
function stubFetch(handler: Handler): void {
calls = [];
vi.stubGlobal(
'fetch',
vi.fn(async (input: unknown, init?: RequestInit) => {
const call: Call = {
url: String(input),
method: init?.method,
headers: (init?.headers ?? {}) as Record<string, string>,
body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined,
};
calls.push(call);
const r = await handler(call, calls.length - 1);
if (r instanceof Error) throw r;
return r;
}),
);
}
/** Replies with the given thunks in order (the last one repeats). */
function queue(...items: Array<() => Reply>): Handler {
return (_c, n) => items[Math.min(n, items.length - 1)]!();
}
afterEach(() => {
vi.unstubAllGlobals();
vi.restoreAllMocks();
});
async function drain<T>(it: AsyncIterable<T>): Promise<T[]> {
const out: T[] = [];
for await (const x of it) out.push(x);
return out;
}
async function run(adapter: ProviderAdapter, cfg: ProviderConfig, req: ChatRequest) {
const events = await drain(adapter.stream(cfg, req));
const done = events[events.length - 1];
if (!done || done.type !== 'done') throw new Error('stream did not end with a done event');
return { events, done };
}
async function failure(p: Promise<unknown>): Promise<AIError> {
const err = await p.then(
() => undefined,
(e: unknown) => e,
);
expect(err).toBeInstanceOf(AIError);
return err as AIError;
}
const TOOL: ToolDef = {
name: 'lookup',
description: 'Look things up',
parameters: { type: 'object', properties: { q: { type: 'string' } }, required: ['q'] },
};
const FRAGS = ['{"q":"ca', 'ts","n"', ':2}'];
const user = (text: string): ChatMessage => ({ role: 'user', content: [{ type: 'text', text }] });
const toolCallMsg = (extra: Partial<ChatMessage> = {}): ChatMessage => ({
role: 'assistant',
content: [{ type: 'tool_call', id: 'call_1', name: 'lookup', input: { q: 'cats' } }],
...extra,
});
const toolResultMsg: ChatMessage = { role: 'user', content: [{ type: 'tool_result', toolCallId: 'call_1', name: 'lookup', output: 'result text' }] };
const cfgOf = (kind: ProviderKind, apiKey: string | null, baseUrl: string | null = null): ProviderConfig => ({ id: kind, kind, name: 'Test Server', baseUrl, apiKey });
/* ------------------------------------------------------------------ */
/* Per-adapter fixtures */
/* ------------------------------------------------------------------ */
const EXPECTED_FULL_USAGE: Usage = { inputTokens: 100, outputTokens: 50, cacheReadTokens: 20, reasoningTokens: 10 };
interface Kit {
name: string;
adapter: ProviderAdapter;
key: string;
cfg: ProviderConfig;
model: string;
otherKind: ProviderKind;
streamUrl: string;
toolId: unknown;
assertAuth(h: Record<string, string>): void;
assertRequest(body: any): void; // system prompt + tool definitions
assertHistory(body: any): void; // tool_call + tool_result mapping
text(parts: string[]): string[];
tool(frags: string[]): string[];
usage(): { chunks: string[]; expected: Usage };
errors: { auth: [number, unknown]; rate: [number, unknown]; ctx: [number, unknown] };
}
const openaiStyleErrors = (key: string): Kit['errors'] => ({
auth: [401, { error: { message: `Incorrect API key provided: ${key}.`, type: 'invalid_request_error', code: 'invalid_api_key' } }],
rate: [429, { error: { message: 'Rate limit reached for requests', type: 'rate_limit_error', code: 'rate_limit_exceeded' } }],
ctx: [400, { error: { message: 'This model maximum context length is 8192 tokens, request too long', type: 'invalid_request_error', code: 'context_length_exceeded' } }],
});
/* ---- Anthropic ---- */
const anthropicKit: Kit = {
name: 'anthropic',
adapter: anthropicAdapter,
key: 'sk-ant-TESTKEY-1234567890',
cfg: cfgOf('anthropic', 'sk-ant-TESTKEY-1234567890'),
model: 'claude-sonnet-5-5',
otherKind: 'openai',
streamUrl: 'https://api.anthropic.com/v1/messages',
toolId: 'call_1',
assertAuth(h) {
expect(h['x-api-key']).toBe('sk-ant-TESTKEY-1234567890');
expect(h['anthropic-version']).toBe('2026-10-21');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.system).toBe('SYS');
expect(b.messages.some((m: any) => m.role === 'system')).toBe(false);
expect(b.tools).toEqual([{ name: 'lookup', description: 'Look things up', input_schema: TOOL.parameters, eager_input_streaming: true }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
{ role: 'assistant', content: [{ type: 'tool_use', id: 'call_1', name: 'lookup', input: { q: 'cats' } }] },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'result text' }] },
]);
},
text: (parts) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }, 'message_start'),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, 'content_block_start'),
...parts.map((p) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: p } }, 'content_block_delta')),
ev({ type: 'content_block_stop', index: 0 }, 'content_block_stop'),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 3 } }, 'message_delta'),
ev({ type: 'message_stop' }, 'message_stop'),
],
tool: (frags) => [
ev({ type: 'message_start', message: { usage: { input_tokens: 7, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'tool_use', id: 'call_1', name: 'lookup', input: {} } }),
...frags.map((f) => ev({ type: 'content_block_delta', index: 0, delta: { type: 'input_json_delta', partial_json: f } })),
ev({ type: 'content_block_stop', index: 0 }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
],
// Anthropic reports uncached/cache-read/cache-write input separately and has no reasoning-token field.
usage: () => ({
chunks: [
ev({ type: 'message_start', message: { usage: { input_tokens: 10, cache_read_input_tokens: 5, cache_creation_input_tokens: 3, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'text', text: 'ok' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 20 } }),
ev({ type: 'message_stop' }),
],
expected: { inputTokens: 18, outputTokens: 20, cacheReadTokens: 5, cacheWriteTokens: 3 },
}),
errors: {
auth: [401, { type: 'error', error: { type: 'authentication_error', message: 'invalid x-api-key sk-ant-TESTKEY-1234567890' } }],
rate: [429, { type: 'error', error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit' } }],
ctx: [400, { type: 'error', error: { type: 'invalid_request_error', message: 'prompt is too long: 250000 tokens > 200000 maximum' } }],
},
};
/* ---- OpenAI (Responses API) ---- */
const oaMessage = (text: string) => ({ type: 'message', id: 'msg_1', role: 'assistant', content: [{ type: 'output_text', text }] });
const openaiKit: Kit = {
name: 'openai',
adapter: openaiAdapter,
key: 'sk-proj-TESTKEY-1234567890',
cfg: cfgOf('openai', 'sk-proj-TESTKEY-1234567890'),
model: 'gpt-4o',
otherKind: 'anthropic',
streamUrl: 'https://api.openai.com/v1/responses',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe('Bearer sk-proj-TESTKEY-1234567890');
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.store).toBe(false);
expect(b.instructions).toBe('SYS');
expect(b.tools).toEqual([{ type: 'function', name: 'lookup', description: 'Look things up', parameters: TOOL.parameters, strict: false }]);
},
assertHistory(b) {
expect(b.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
{ type: 'function_call', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
},
text: (parts) => [
...parts.map((p) => ev({ type: 'response.output_text.delta', item_id: 'msg_1', delta: p }, 'response.output_text.delta')),
ev({ type: 'response.completed', response: { output: [oaMessage(parts.join(''))], usage: { input_tokens: 4, output_tokens: 3 } } }, 'response.completed'),
],
// `arguments` is empty in the final item here so that the streamed deltas are what gets assembled.
tool: (frags) => [
...frags.map((d) => ev({ type: 'response.function_call_arguments.delta', item_id: 'fc_1', delta: d })),
ev({
type: 'response.completed',
response: { output: [{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '' }], usage: { input_tokens: 4, output_tokens: 3 } },
}),
],
usage: () => ({
chunks: [
ev({ type: 'response.output_text.delta', delta: 'ok' }),
ev({
type: 'response.completed',
response: {
output: [oaMessage('ok')],
usage: { input_tokens: 100, output_tokens: 50, input_tokens_details: { cached_tokens: 20 }, output_tokens_details: { reasoning_tokens: 10 } },
},
}),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors('sk-proj-TESTKEY-1234567890'),
};
/* ---- Gemini ---- */
const gemChunk = (parts: unknown[], extra: Record<string, unknown> = {}, cand: Record<string, unknown> = {}) =>
ev({ candidates: [{ content: { role: 'model', parts }, ...cand }], ...extra });
const geminiKit: Kit = {
name: 'gemini',
adapter: geminiAdapter,
key: 'AIzaSyTESTKEY1234567890abcd',
cfg: cfgOf('gemini', 'AIzaSyTESTKEY1234567890abcd'),
model: 'gemini-3.8-flash',
otherKind: 'anthropic',
streamUrl: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse',
toolId: expect.stringMatching(/^gem_/),
assertAuth(h) {
expect(h['x-goog-api-key']).toBe('AIzaSyTESTKEY1234567890abcd');
expect(h.Authorization).toBeUndefined();
},
assertRequest(b) {
expect(b.systemInstruction).toEqual({ parts: [{ text: 'SYS' }] });
expect(b.tools).toEqual([{ functionDeclarations: [{ name: 'lookup', description: 'Look things up', parameters: TOOL.parameters }] }]);
},
assertHistory(b) {
expect(b.contents).toEqual([
{ role: 'user', parts: [{ text: 'hi' }] },
{ role: 'model', parts: [{ functionCall: { name: 'lookup', args: { q: 'cats' } } }] },
{ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] },
]);
},
text: (parts) => [...parts.map((p) => gemChunk([{ text: p }])), gemChunk([{ text: '' }], {}, { finishReason: 'STOP' })],
// Gemini delivers a function call whole (args is an object), never as fragments.
tool: (frags) => [gemChunk([{ functionCall: { name: 'lookup', args: JSON.parse(frags.join('')) } }], {}, { finishReason: 'STOP' })],
usage: () => ({
chunks: [
gemChunk([{ text: 'ok' }], { usageMetadata: { promptTokenCount: 100, candidatesTokenCount: 40, thoughtsTokenCount: 10, cachedContentTokenCount: 20 } }, { finishReason: 'STOP' }),
],
expected: EXPECTED_FULL_USAGE,
}),
errors: {
auth: [401, { error: { code: 401, message: 'API key not valid. AIzaSyTESTKEY1234567890abcd', status: 'UNAUTHENTICATED' } }],
rate: [429, { error: { code: 429, message: 'Resource has been exhausted', status: 'RESOURCE_EXHAUSTED' } }],
ctx: [400, { error: { code: 400, message: 'The input token count (2000000) exceeds the maximum number of tokens allowed (1048576).', status: 'INVALID_ARGUMENT' } }],
},
};
/* ---- Chat Completions family ---- */
const ccChunk = (delta: Record<string, unknown>, finish: string | null = null) => ev({ choices: [{ index: 0, delta, finish_reason: finish }] });
function chatCompletionsKit(over: Pick<Kit, 'name' | 'adapter' | 'key' | 'cfg' | 'model' | 'streamUrl'>): Kit {
return {
...over,
otherKind: 'gemini',
toolId: 'call_1',
assertAuth(h) {
expect(h.Authorization).toBe(`Bearer ${over.key}`);
},
assertRequest(b) {
expect(b.stream).toBe(true);
expect(b.messages[0]).toEqual({ role: 'system', content: 'SYS' });
expect(b.tools).toEqual([{ type: 'function', function: { name: 'lookup', description: 'Look things up', parameters: TOOL.parameters } }]);
},
assertHistory(b) {
expect(b.messages).toEqual([
{ role: 'system', content: 'SYS' },
{ role: 'user', content: 'hi' },
{ role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '{"q":"cats"}' } }] },
{ role: 'tool', tool_call_id: 'call_1', content: 'result text' },
]);
},
text: (parts) => [ccChunk({ role: 'assistant', content: '' }), ...parts.map((p) => ccChunk({ content: p })), ccChunk({}, 'stop'), DONE],
tool: (frags) => [
ccChunk({ role: 'assistant', tool_calls: [{ index: 0, id: 'call_1', type: 'function', function: { name: 'lookup', arguments: '' } }] }),
...frags.map((f) => ccChunk({ tool_calls: [{ index: 0, function: { arguments: f } }] })),
ccChunk({}, 'tool_calls'),
DONE,
],
usage: () => ({
chunks: [
ccChunk({ content: 'ok' }, 'stop'),
ev({
choices: [],
usage: { prompt_tokens: 100, completion_tokens: 50, prompt_tokens_details: { cached_tokens: 20 }, completion_tokens_details: { reasoning_tokens: 10 } },
}),
DONE,
],
expected: EXPECTED_FULL_USAGE,
}),
errors: openaiStyleErrors(over.key),
};
}
const compatKit = chatCompletionsKit({
name: 'openai_compatible',
adapter: openaiCompatibleAdapter,
key: 'sk-or-TESTKEY-1234567890',
cfg: cfgOf('openai_compatible', 'sk-or-TESTKEY-1234567890', 'https://router.example.com/api/v1'),
model: 'meta/llama-3',
streamUrl: 'https://router.example.com/api/v1/chat/completions',
});
const ollamaKit = chatCompletionsKit({
name: 'ollama',
adapter: ollamaAdapter,
key: 'ollama-proxy-secret-1234',
cfg: cfgOf('ollama', 'ollama-proxy-secret-1234'),
model: 'llama3',
streamUrl: 'http://localhost:11434/v1/chat/completions',
});
const kits = [anthropicKit, openaiKit, geminiKit, compatKit, ollamaKit];
/* ------------------------------------------------------------------ */
/* Shared behaviour across all five adapters */
/* ------------------------------------------------------------------ */
describe.each(kits)('$name adapter', (kit) => {
const baseReq = (): ChatRequest => ({ model: kit.model, system: 'SYS', messages: [user('hi')], tools: [TOOL] });
it('sends the expected URL, auth header, system prompt and tool definitions; key stays out of the URL', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, baseReq());
expect(calls).toHaveLength(1);
expect(calls[0]!.url).toBe(kit.streamUrl);
expect(calls[0]!.url).not.toContain(kit.key);
expect(calls[0]!.method).toBe('POST');
kit.assertAuth(calls[0]!.headers);
kit.assertRequest(calls[0]!.body);
});
it('maps tool_call and tool_result history into the provider format', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg(), toolResultMsg] });
kit.assertHistory(calls[0]!.body);
});
it('rebuilds from neutral content when providerData belongs to a different provider kind', async () => {
stubFetch(queue(() => sseResponse(kit.text(['ok']))));
// The foreign payload carries every shape key so it would leak through if the kind were not checked.
const foreign = { kind: kit.otherKind, data: { content: [{ type: 'text', text: 'MARKER' }], output: [{ type: 'message', marker: 'MARKER' }], parts: [{ text: 'MARKER' }] } };
await run(kit.adapter, kit.cfg, { ...baseReq(), messages: [user('hi'), toolCallMsg({ providerData: foreign }), toolResultMsg] });
expect(JSON.stringify(calls[0]!.body)).not.toContain('MARKER');
kit.assertHistory(calls[0]!.body);
});
it('streams text events in order and concatenates them in done', async () => {
stubFetch(queue(() => sseResponse(kit.text(['Hel', 'lo', ' world']))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
expect(events.filter((e) => e.type === 'text').map((e) => (e as { delta: string }).delta)).toEqual(['Hel', 'lo', ' world']);
expect(done.message.role).toBe('assistant');
expect(done.message.content).toEqual([{ type: 'text', text: 'Hello world' }]);
expect(done.stopReason).toBe('end');
});
it('assembles a fragmented tool call into one tool_call event', async () => {
stubFetch(queue(() => sseResponse(kit.tool(FRAGS))));
const { events, done } = await run(kit.adapter, kit.cfg, baseReq());
const toolEvents = events.filter((e): e is Extract<StreamEvent, { type: 'tool_call' }> => e.type === 'tool_call');
expect(toolEvents).toHaveLength(1);
expect(toolEvents[0]!.call).toEqual({ id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(done.stopReason).toBe('tool_use');
expect(done.message.content).toContainEqual({ type: 'tool_call', id: kit.toolId, name: 'lookup', input: { q: 'cats', n: 2 } });
expect(events[events.length - 1]).toBe(done);
});
it('maps usage onto the neutral convention', async () => {
const { chunks, expected } = kit.usage();
stubFetch(queue(() => sseResponse(chunks)));
const { done } = await run(kit.adapter, kit.cfg, baseReq());
expect(done.usage).toEqual(expected);
});
it.each([
['auth', 'auth', 401],
['rate', 'rate_limit', 429],
['ctx', 'context_length', 400],
] as const)('maps HTTP %s failure to AIError code %s', async (which, code, status) => {
const [st, body] = kit.errors[which];
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe(code);
expect(err.status).toBe(status);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
it('scrubs the API key from provider text', async () => {
const [st, body] = kit.errors.auth;
stubFetch(queue(() => jsonResponse(body, st)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.providerMessage).toContain('[redacted]');
expect(JSON.stringify(err)).not.toContain(kit.key);
});
it('maps a fetch TypeError to a network error without leaking the key', async () => {
stubFetch(queue(() => new TypeError(`fetch failed (${kit.key})`)));
const err = await failure(run(kit.adapter, kit.cfg, baseReq()));
expect(err.code).toBe('network');
expect(err.retryable).toBe(true);
expect(err.message).not.toContain(kit.key);
expect(err.providerMessage ?? '').not.toContain(kit.key);
});
});
/* ------------------------------------------------------------------ */
/* Anthropic specifics */
/* ------------------------------------------------------------------ */
describe('anthropic providerData', () => {
const thinkingTurn = [
ev({ type: 'message_start', message: { usage: { input_tokens: 5, output_tokens: 1 } } }),
ev({ type: 'content_block_start', index: 0, content_block: { type: 'thinking', thinking: '', signature: '' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'Let me ' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'thinking_delta', thinking: 'think' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'sig-' } }),
ev({ type: 'content_block_delta', index: 0, delta: { type: 'signature_delta', signature: 'abc' } }),
ev({ type: 'content_block_start', index: 1, content_block: { type: 'text', text: '' } }),
ev({ type: 'content_block_delta', index: 1, delta: { type: 'text_delta', text: 'Sure' } }),
ev({ type: 'content_block_start', index: 2, content_block: { type: 'tool_use', id: 'toolu_1', name: 'lookup', input: {} } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '{"q":' } }),
ev({ type: 'content_block_delta', index: 2, delta: { type: 'input_json_delta', partial_json: '"cats"}' } }),
ev({ type: 'message_delta', delta: { stop_reason: 'tool_use' }, usage: { output_tokens: 9 } }),
ev({ type: 'message_stop' }),
];
const replayed = [
{ type: 'thinking', thinking: 'Let me think', signature: 'sig-abc' },
{ type: 'text', text: 'Sure' },
{ type: 'tool_use', id: 'toolu_1', name: 'lookup', input: { q: 'cats' } },
];
const followUp = (assistant: ChatMessage): ChatRequest => ({
model: 'claude-sonnet-5-5',
messages: [user('hi'), assistant, { role: 'user', content: [{ type: 'tool_result', toolCallId: 'toolu_1', name: 'lookup', output: 'r' }] }],
});
it('requests adaptive thinking, emits reasoning, and replays thinking blocks with signatures', async () => {
stubFetch(queue(() => sseResponse(thinkingTurn), () => sseResponse(anthropicKit.text(['done']))));
const { events, done } = await run(anthropicAdapter, anthropicKit.cfg, { model: 'claude-sonnet-5-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'high' });
expect(calls[0]!.body.thinking).toEqual({ type: 'adaptive', display: 'summarized' });
expect(calls[0]!.body.output_config).toEqual({ effort: 'high' });
expect(events.filter((e) => e.type === 'reasoning').map((e) => (e as { delta: string }).delta)).toEqual(['Let me ', 'think']);
expect(done.message.providerData).toEqual({ kind: 'anthropic', data: { content: replayed } });
await run(anthropicAdapter, anthropicKit.cfg, followUp(done.message));
expect(calls[1]!.body.messages[1]).toEqual({ role: 'assistant', content: replayed });
expect(calls[1]!.body.messages[2].content[0]).toMatchObject({ type: 'tool_result', tool_use_id: 'toolu_1', content: 'r' });
});
it('strips thinking blocks and retries once on an invalid-signature 400', async () => {
const msg: ChatMessage = { ...toolCallMsg(), providerData: { kind: 'anthropic', data: { content: replayed } } };
stubFetch(
queue(
() => jsonResponse({ type: 'error', error: { type: 'invalid_request_error', message: 'Invalid `signature` in `thinking` block' } }, 400),
() => sseResponse(anthropicKit.text(['ok'])),
),
);
await run(anthropicAdapter, anthropicKit.cfg, followUp(msg));
expect(calls).toHaveLength(2);
expect(JSON.stringify(calls[0]!.body)).toContain('"thinking"');
expect(calls[1]!.body.messages[1].content.map((b: any) => b.type)).toEqual(['text', 'tool_use']);
});
});
describe('anthropic listModels', () => {
it('paginates, filters non-chat ids and merges API data over catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
data: [
{
id: 'claude-haiku-4-5',
display_name: 'Haiku API Name',
max_input_tokens: 200000,
max_tokens: 64000,
capabilities: { image_input: { supported: true }, pdf_input: { supported: false }, thinking: { supported: true } },
},
{ id: 'claude-embed-1' },
],
has_more: true,
last_id: 'claude-haiku-4-5',
}),
() => jsonResponse({ data: [{ id: 'claude-unknown-9' }], has_more: false }),
),
);
const models = await anthropicAdapter.listModels(anthropicKit.cfg);
expect(calls[0]!.url).toBe('https://api.anthropic.com/v1/models?limit=1000');
expect(calls[1]!.url).toContain('after_id=claude-haiku-4-5');
expect(calls[0]!.url).not.toContain(anthropicKit.key);
anthropicKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['claude-haiku-4-5', 'claude-unknown-9']);
expect(models[0]).toMatchObject({
label: 'Haiku API Name', // reported by the API, wins over the catalog
contextWindow: 200000,
maxOutput: 64000,
priceIn: 1, // from the catalog
priceOut: 5,
capabilities: { tools: true, vision: true, pdf: false, reasoning: true, temperature: false },
});
expect(models[1]).toEqual({ id: 'claude-unknown-9', capabilities: { tools: true, temperature: false } });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI specifics */
/* ------------------------------------------------------------------ */
describe('openai providerData and reasoning', () => {
const output = [
{ type: 'reasoning', id: 'rs_1', encrypted_content: 'ENC', summary: [] },
oaMessage('Hi'),
{ type: 'function_call', id: 'fc_1', call_id: 'call_1', name: 'lookup', arguments: '{"q":"cats"}' },
];
const turn = [
ev({ type: 'response.reasoning_summary_text.delta', delta: 'Thinking' }),
ev({ type: 'response.output_text.delta', delta: 'Hi' }),
ev({ type: 'response.completed', response: { output, usage: { input_tokens: 3, output_tokens: 2 } } }),
];
it('asks for encrypted reasoning and replays output items (reasoning item included) verbatim', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(openaiKit.text(['ok']))));
const { events, done } = await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.2 });
const body = calls[0]!.body;
expect(body.reasoning).toEqual({ effort: 'medium', summary: 'auto' });
expect(body.include).toEqual(['reasoning.encrypted_content']);
expect(body.temperature).toBeUndefined(); // reasoning models reject it
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Thinking' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'openai', data: { output } });
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.input).toEqual([
{ role: 'user', content: [{ type: 'input_text', text: 'hi' }] },
...output,
{ type: 'function_call_output', call_id: 'call_1', output: 'result text' },
]);
});
it('retries once without reasoning.effort when the model rejects it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unsupported value: 'reasoning.effort' does not support 'none'", type: 'invalid_request_error' } }, 400),
() => sseResponse(openaiKit.text(['ok'])),
),
);
await run(openaiAdapter, openaiKit.cfg, { model: 'gpt-5', messages: [user('hi')], reasoningEffort: 'none' });
expect(calls[0]!.body.reasoning.effort).toBe('none');
expect(calls[1]!.body.reasoning.effort).toBe('minimal');
});
});
describe('openai listModels', () => {
it('keeps chat models only, sorts them and merges catalog defaults', async () => {
const ids = ['gpt-4o', 'gpt-4o-mini', 'o3', 'text-embedding-3-large', 'whisper-1', 'dall-e-3', 'gpt-4o-mini-tts', 'gpt-4o-realtime-preview', 'gpt-5-search-api', 'davinci-002', 'gpt-3.5-turbo-instruct', 'omni-moderation-latest'];
stubFetch(queue(() => jsonResponse({ data: ids.map((id) => ({ id })) })));
const models = await openaiAdapter.listModels(openaiKit.cfg);
expect(calls[0]!.url).toBe('https://api.openai.com/v1/models');
openaiKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id).sort()).toEqual(['gpt-4o', 'gpt-4o-mini', 'o3']);
expect(models.find((m) => m.id === 'gpt-4o')).toMatchObject({ label: 'GPT-4o', contextWindow: 128000, priceIn: 2.5, priceOut: 10, capabilities: { reasoning: false, temperature: true } });
expect(models.find((m) => m.id === 'o3')?.capabilities).toMatchObject({ reasoning: true, temperature: false });
});
});
/* ------------------------------------------------------------------ */
/* Gemini specifics */
/* ------------------------------------------------------------------ */
describe('gemini providerData and request config', () => {
const turn = [
gemChunk([{ text: 'Pondering', thought: true, thoughtSignature: 'SIGTHOUGHT' }]),
gemChunk([{ text: 'Hello' }]),
gemChunk([{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' }], { usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 2 } }, { finishReason: 'STOP' }),
];
const replayed = [
{ text: '', thought: true, thoughtSignature: 'SIGTHOUGHT' },
{ text: 'Hello' },
{ functionCall: { name: 'lookup', args: { q: 'cats' } }, thoughtSignature: 'SIGFN' },
];
it('keeps thought signatures in providerData and replays the raw parts', async () => {
stubFetch(queue(() => sseResponse(turn), () => sseResponse(geminiKit.text(['ok']))));
const { events, done } = await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')], tools: [TOOL], reasoningEffort: 'medium', temperature: 0.3, maxOutputTokens: 100 });
expect(calls[0]!.body.generationConfig).toEqual({ temperature: 0.3, maxOutputTokens: 100, thinkingConfig: { includeThoughts: true, thinkingLevel: 'medium' } });
expect(events.filter((e) => e.type === 'reasoning')).toEqual([{ type: 'reasoning', delta: 'Pondering' }]);
expect(done.stopReason).toBe('tool_use');
expect(done.message.providerData).toEqual({ kind: 'gemini', data: { parts: replayed } });
await run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi'), done.message, toolResultMsg] });
expect(calls[1]!.body.contents[1]).toEqual({ role: 'model', parts: replayed });
expect(calls[1]!.body.contents[2]).toEqual({ role: 'user', parts: [{ functionResponse: { name: 'lookup', response: { result: 'result text' } } }] });
});
it('turns a blocked prompt into a content_filter error', async () => {
stubFetch(queue(() => sseResponse([ev({ promptFeedback: { blockReason: 'SAFETY' } })])));
const err = await failure(run(geminiAdapter, geminiKit.cfg, { model: 'gemini-3.8-flash', messages: [user('hi')] }));
expect(err.code).toBe('content_filter');
});
});
describe('gemini listModels', () => {
it('paginates, keeps generateContent chat models and merges catalog defaults', async () => {
stubFetch(
queue(
() =>
jsonResponse({
models: [
{ name: 'models/gemini-3.8-flash', displayName: 'Gemini API Name', inputTokenLimit: 1000000, outputTokenLimit: 65536, supportedGenerationMethods: ['generateContent'], thinking: true },
{ name: 'models/gemini-embedding-001', supportedGenerationMethods: ['embedContent'] },
{ name: 'models/gemini-2.5-flash-image', supportedGenerationMethods: ['generateContent'] },
],
nextPageToken: 'tok1',
}),
() => jsonResponse({ models: [{ name: 'models/gemma-3-27b-it', supportedGenerationMethods: ['generateContent'] }, { name: 'models/text-bison-001', supportedGenerationMethods: ['generateContent'] }] }),
),
);
const models = await geminiAdapter.listModels(geminiKit.cfg);
expect(calls[0]!.url).toBe('https://generativelanguage.googleapis.com/v1beta/models?pageSize=1000');
expect(calls[1]!.url).toContain('pageToken=tok1');
for (const c of calls) {
expect(c.url).not.toContain(geminiKit.key);
geminiKit.assertAuth(c.headers);
}
expect(models.map((m) => m.id)).toEqual(['gemini-3.8-flash', 'gemma-3-27b-it']);
expect(models[0]).toMatchObject({ label: 'Gemini API Name', contextWindow: 1000000, maxOutput: 65536, priceIn: 0.75, priceOut: 3.75, capabilities: { reasoning: true, tools: true } });
expect(models[1]!.capabilities).toMatchObject({ tools: false, pdf: false });
});
});
describe('toGeminiSchema', () => {
it('strips unsupported keywords and converts nullable/enum/object shapes', () => {
const out = toGeminiSchema({
$schema: 'http://json-schema.org/draft-07/schema#',
type: 'object',
additionalProperties: false,
properties: {
name: { type: ['string', 'null'], default: 'a', pattern: '^a', format: 'email' },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string', const: 'x' }, minItems: 1, uniqueItems: true },
mode: { enum: ['a', 'b'] },
nested: { type: 'object' },
},
required: ['name'],
});
expect(out).toEqual({
type: 'object',
properties: {
name: { type: 'string', nullable: true },
when: { type: 'string', format: 'date-time' },
tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
mode: { enum: ['a', 'b'], type: 'string' },
nested: { type: 'object', properties: {} },
},
required: ['name'],
});
});
it('falls back to a string schema for non-objects', () => {
expect(toGeminiSchema('nope')).toEqual({ type: 'string' });
});
});
/* ------------------------------------------------------------------ */
/* OpenAI-compatible and Ollama specifics */
/* ------------------------------------------------------------------ */
describe('openai-compatible stream_options fallback', () => {
it('retries without stream_options when the first 400 complains about it', async () => {
stubFetch(
queue(
() => jsonResponse({ error: { message: "Unknown parameter: 'stream_options'.", type: 'invalid_request_error' } }, 400),
() => sseResponse(compatKit.text(['ok'])),
),
);
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(calls).toHaveLength(2);
expect(calls[0]!.body.stream_options).toEqual({ include_usage: true });
expect(calls[1]!.body.stream_options).toBeUndefined();
expect(done.message.content).toEqual([{ type: 'text', text: 'ok' }]);
});
it('does not write providerData (it only speaks neutral content)', async () => {
stubFetch(queue(() => sseResponse(compatKit.text(['ok']))));
const { done } = await run(compatKit.adapter, compatKit.cfg, { model: compatKit.model, messages: [user('hi')] });
expect(done.message.providerData).toBeUndefined();
});
});
describe('openai-compatible listModels', () => {
it('parses OpenRouter-style metadata, filters embeddings and merges vendor catalog defaults', async () => {
stubFetch(
queue(() =>
jsonResponse({
data: [
{
id: 'anthropic/claude-sonnet-5-5',
name: 'Claude Sonnet 5.5',
context_length: 1000000,
top_provider: { max_completion_tokens: 64000 },
pricing: { prompt: '0.000002', completion: '0.00001' },
supported_parameters: ['tools', 'reasoning'],
architecture: { input_modalities: ['text', 'image'] },
},
{ id: 'openai/text-embedding-3-small' },
{ id: 'vendor/plain' },
],
}),
),
);
const models = await openaiCompatibleAdapter.listModels(compatKit.cfg);
expect(calls[0]!.url).toBe('https://router.example.com/api/v1/models');
compatKit.assertAuth(calls[0]!.headers);
expect(models.map((m) => m.id)).toEqual(['anthropic/claude-sonnet-5-5', 'vendor/plain']);
expect(models[0]).toMatchObject({
label: 'Claude Sonnet 5.5',
contextWindow: 1000000,
maxOutput: 64000,
priceIn: 2,
priceOut: 10,
capabilities: { tools: true, reasoning: true, vision: true, pdf: true }, // pdf borrowed from the Anthropic catalog entry
});
expect(models[0]!.capabilities).not.toHaveProperty('temperature');
expect(models[1]).toEqual({ id: 'vendor/plain' });
});
it('reports "unsupported" when the server has no /models endpoint', async () => {
stubFetch(queue(() => jsonResponse({ error: 'not found' }, 404)));
const err = await failure(openaiCompatibleAdapter.listModels(compatKit.cfg));
expect(err.code).toBe('unsupported');
});
});
describe('ollama listModels', () => {
it('uses /api/tags + /api/show, drops embedding-only models and keeps models whose /show fails', async () => {
stubFetch((c) => {
if (c.url.endsWith('/api/tags')) {
return jsonResponse({
models: [
{ name: 'old:1b', model: 'old:1b' },
{ name: 'nomic-embed-text:latest', model: 'nomic-embed-text:latest' },
{ name: 'llama3:latest', model: 'llama3:latest' },
],
});
}
if (c.url.endsWith('/api/show')) {
if (c.body.model === 'llama3:latest') return jsonResponse({ capabilities: ['completion', 'tools'], model_info: { 'llama.context_length': 8192 } });
if (c.body.model === 'nomic-embed-text:latest') return jsonResponse({ capabilities: ['embedding'] });
return jsonResponse({ error: 'unsupported' }, 404);
}
return new Response('?', { status: 404 });
});
const models = await ollamaAdapter.listModels(ollamaKit.cfg);
expect(calls[0]!.url).toBe('http://localhost:11434/api/tags');
expect(calls.filter((c) => c.url.endsWith('/api/show')).map((c) => c.method)).toEqual(['POST', 'POST', 'POST']);
ollamaKit.assertAuth(calls[0]!.headers);
expect(models).toEqual([
{
id: 'llama3:latest',
label: 'llama3:latest',
priceIn: 0,
priceOut: 0,
contextWindow: 8192,
capabilities: { tools: true, vision: false, reasoning: false, pdf: false, temperature: true },
},
{ id: 'old:1b', label: 'old:1b', priceIn: 0, priceOut: 0 },
]);
});
it('strips /v1 from a configured base URL', async () => {
stubFetch(queue(() => jsonResponse({ models: [] })));
await ollamaAdapter.listModels({ ...ollamaKit.cfg, baseUrl: 'http://box:11434/v1/' });
expect(calls[0]!.url).toBe('http://box:11434/api/tags');
});
});
describe('normalizeCompatBase', () => {
it.each([
['https://api.x.com/v1', 'https://api.x.com/v1'],
['https://api.x.com/v1/', 'https://api.x.com/v1'],
[' https://x.com/api/v1 ', 'https://x.com/api/v1'],
['https://api.x.com/v1/chat/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/completions', 'https://api.x.com/v1'],
['https://api.x.com/v1/models', 'https://api.x.com/v1'],
['http://localhost:1234', 'http://localhost:1234/v1'],
['http://localhost:1234/', 'http://localhost:1234/v1'],
])('%s -> %s', (input, expected) => {
expect(normalizeCompatBase(input)).toBe(expected);
});
});
/* ------------------------------------------------------------------ */
/* catalog */
/* ------------------------------------------------------------------ */
describe('lookupModelDefaults', () => {
it('matches exact ids', () => {
expect(lookupModelDefaults('anthropic', 'claude-haiku-4-5')).toMatchObject({ label: 'Claude Haiku 4.5', priceIn: 1, priceOut: 5, contextWindow: 200000 });
});
it('matches dated snapshots by prefix, without the base label', () => {
const d = lookupModelDefaults('anthropic', 'claude-haiku-4-5-20251001');
expect(d).toMatchObject({ priceIn: 1, contextWindow: 200000 });
expect(d.label).toBeUndefined();
expect(lookupModelDefaults('openai', 'gpt-5-2025-08-07')).toMatchObject({ priceIn: 1.25 });
expect(lookupModelDefaults('gemini', 'models/gemini-2.5-flash-latest')).toMatchObject({ priceIn: 0.3 });
});
it('keeps capabilities but drops prices for a different variant of a known model', () => {
const d = lookupModelDefaults('openai', 'o3-deep-research');
expect(d.capabilities?.reasoning).toBe(true);
expect(d.priceIn).toBeUndefined();
expect(d.priceOut).toBeUndefined();
});
it('returns {} for unknown models and for kinds without a table', () => {
expect(lookupModelDefaults('openai', 'totally-unknown-model')).toEqual({});
expect(lookupModelDefaults('anthropic', '')).toEqual({});
expect(lookupModelDefaults('ollama', 'llama3')).toEqual({});
});
it('borrows capabilities but not prices or temperature for router ids', () => {
const d = lookupModelDefaults('openai_compatible', 'anthropic/claude-sonnet-5-5');
expect(d.priceIn).toBeUndefined();
expect(d.capabilities).toMatchObject({ tools: true, vision: true, pdf: true });
expect(d.capabilities).not.toHaveProperty('temperature');
});
it('returns copies that are safe to mutate', () => {
const a = lookupModelDefaults('openai', 'gpt-4o');
a.capabilities!.tools = false;
expect(lookupModelDefaults('openai', 'gpt-4o').capabilities!.tools).toBe(true);
});
});
/* ------------------------------------------------------------------ */
/* parseSSE */
/* ------------------------------------------------------------------ */
describe('parseSSE', () => {
const parse = (chunks: string[]) => drain(parseSSE(textStream(chunks)));
it('handles CRLF line endings', async () => {
expect(await parse(['event: ping\r\ndata: {"a":1}\r\n\r\ndata: two\r\n\r\n'])).toEqual([{ event: 'ping', data: '{"a":1}' }, { data: 'two' }]);
});
it('joins multi-line data with newlines', async () => {
expect(await parse(['data: a\ndata: b\n\n'])).toEqual([{ data: 'a\nb' }]);
});
it('ignores comments and keep-alives', async () => {
expect(await parse([': keepalive\n\ndata: x\n: mid-event comment\n\n'])).toEqual([{ data: 'x' }]);
});
it('stops at [DONE]', async () => {
expect(await parse(['data: a\n\ndata: [DONE]\n\ndata: b\n\n'])).toEqual([{ data: 'a' }]);
});
it('handles chunk boundaries inside a line', async () => {
expect(await parse(['da', 'ta: hel', 'lo\n', '\ndata: wor', 'ld\n\n'])).toEqual([{ data: 'hello' }, { data: 'world' }]);
});
it('handles CRLF split across chunks', async () => {
expect(await parse(['data: x\r', '\n\r', '\ndata: y\n\n'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('handles bare CR line endings', async () => {
expect(await parse(['data: x\r\rdata: y\r\r'])).toEqual([{ data: 'x' }, { data: 'y' }]);
});
it('flushes a final event with no trailing blank line', async () => {
expect(await parse(['data: a\n\ndata: tail'])).toEqual([{ data: 'a' }, { data: 'tail' }]);
});
it('strips a BOM and decodes multi-byte characters split across chunks', async () => {
const bytes = enc.encode('data: €uro\n\n');
const euro = bytes.indexOf(0xe2); // first byte of the 3-byte euro sign
const stream = bytesStream([bytes.slice(0, euro + 1), bytes.slice(euro + 1, euro + 2), bytes.slice(euro + 2)]);
expect(await drain(parseSSE(stream))).toEqual([{ data: '€uro' }]);
});
});

624
server/ai/service.ts Normal file
View file

@ -0,0 +1,624 @@
/**
* Provider + model configuration, task routing, usage accounting and the two call
* primitives everything else builds on: `runChat` (one streamed model turn) and
* `generateStructured` (schema-constrained JSON through a forced tool call).
*/
import { jsonrepair } from 'jsonrepair';
import { z } from 'zod';
import { decryptSecret, encryptSecret, maskSecret } from '../crypto.ts';
import type { DB } from '../db.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { getSetting, setSetting } from '../settings.ts';
import { newId, now, parseJson } from '../util.ts';
import { getAdapter, lookupModelDefaults } from './providers/index.ts';
import {
AIError,
DEFAULT_CAPABILITIES,
PROVIDER_KINDS,
type ChatMessage,
type ChatRequest,
type DiscoveredModel,
type ModelCapabilities,
type ProviderConfig,
type ProviderKind,
type ReasoningEffort,
type StopReason,
type StreamEvent,
type ToolDef,
type Usage,
} from './types.ts';
/* ------------------------------ rows + DTOs ------------------------------ */
interface ProviderRow {
id: string;
name: string;
kind: ProviderKind;
base_url: string | null;
api_key_enc: string | null;
headers_enc: string | null;
enabled: number;
last_test_at: number | null;
last_test_ok: number | null;
last_test_message: string;
created_at: number;
updated_at: number;
}
interface ModelRow {
id: string;
provider_id: string;
model: string;
label: string;
source: string;
enabled: number;
capabilities: string;
context_window: number | null;
max_output: number | null;
price_in: number | null;
price_out: number | null;
}
export const TASKS = ['chat', 'generate', 'vision', 'quick'] as const;
export type TaskName = (typeof TASKS)[number];
export const TASK_INFO: Record<TaskName, { label: string; description: string }> = {
chat: { label: 'Assistant', description: 'The assistant that searches your library, answers questions and runs tools.' },
generate: { label: 'Content generation', description: 'Writing quizzes, flashcards and study guides. Benefits most from a strong model.' },
vision: { label: 'Vision and OCR', description: 'Reading scanned PDFs and images. Needs a model that accepts images.' },
quick: { label: 'Quick explanations', description: 'Short explanations while studying. A fast, inexpensive model works well.' },
};
export interface TaskConfig {
modelId: string | null;
temperature: number | null;
reasoningEffort: ReasoningEffort | null;
maxOutputTokens: number | null;
}
export type TaskSettings = Record<TaskName | 'default', TaskConfig>;
const EMPTY_TASK: TaskConfig = { modelId: null, temperature: null, reasoningEffort: null, maxOutputTokens: null };
export const taskConfigSchema = z.object({
modelId: z.string().nullable(),
temperature: z.number().min(0).max(2).nullable(),
reasoningEffort: z.enum(['none', 'low', 'medium', 'high']).nullable(),
maxOutputTokens: z.number().int().min(256).max(200000).nullable(),
});
function mapProvider(app: AppContext, row: ProviderRow) {
const key = decryptSecret(row.api_key_enc, app.config.secretKey);
const headers = parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {});
return {
id: row.id,
name: row.name,
kind: row.kind,
baseUrl: row.base_url,
hasKey: Boolean(row.api_key_enc),
/** Only ever a masked preview; the key itself never leaves the server. */
keyPreview: key ? maskSecret(key) : row.api_key_enc ? 'unreadable (re-enter key)' : '',
headerNames: Object.keys(headers),
enabled: Boolean(row.enabled),
lastTestAt: row.last_test_at,
lastTestOk: row.last_test_ok === null ? null : Boolean(row.last_test_ok),
lastTestMessage: row.last_test_message,
createdAt: row.created_at,
};
}
function mapModel(row: ModelRow & { provider_name?: string; provider_kind?: string; provider_enabled?: number }) {
return {
id: row.id,
providerId: row.provider_id,
providerName: row.provider_name ?? '',
providerKind: (row.provider_kind ?? '') as ProviderKind,
model: row.model,
label: row.label || row.model,
source: row.source,
enabled: Boolean(row.enabled),
available: Boolean(row.enabled) && row.provider_enabled !== 0,
capabilities: { ...DEFAULT_CAPABILITIES, ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}) },
contextWindow: row.context_window,
maxOutput: row.max_output,
priceIn: row.price_in,
priceOut: row.price_out,
};
}
export type ModelDto = ReturnType<typeof mapModel>;
const MODEL_SELECT = `SELECT m.*, p.name AS provider_name, p.kind AS provider_kind, p.enabled AS provider_enabled FROM ai_models m JOIN ai_providers p ON p.id = m.provider_id`;
/* ------------------------------ providers ------------------------------ */
export const providerInputSchema = z.object({
name: z.string().trim().min(1).max(80),
kind: z.enum(PROVIDER_KINDS as [ProviderKind, ...ProviderKind[]]),
baseUrl: z
.string()
.trim()
.max(500)
.nullish()
.transform((v) => v || null)
.refine((v) => v === null || /^https?:\/\//i.test(v), 'URL must start with http:// or https://'),
/** undefined = keep the stored key, '' or null = remove it. */
apiKey: z.string().max(2000).nullish(),
headers: z.record(z.string().max(100), z.string().max(2000)).optional(),
enabled: z.boolean().optional(),
});
function providerConfig(app: AppContext, row: ProviderRow): ProviderConfig {
return {
id: row.id,
kind: row.kind,
name: row.name,
baseUrl: row.base_url,
apiKey: decryptSecret(row.api_key_enc, app.config.secretKey),
headers: parseJson<Record<string, string>>(decryptSecret(row.headers_enc, app.config.secretKey), {}),
};
}
function providerRow(db: DB, id: string): ProviderRow {
const row = db.get<ProviderRow>('SELECT * FROM ai_providers WHERE id = ?', id);
if (!row) throw notFound('Provider');
return row;
}
export function listProviders(app: AppContext) {
return app.db.all<ProviderRow>('SELECT * FROM ai_providers ORDER BY created_at').map((r) => mapProvider(app, r));
}
export function saveProvider(app: AppContext, id: string | null, input: z.input<typeof providerInputSchema>) {
const data = providerInputSchema.parse(input);
const meta = getAdapter(data.kind).meta;
if (meta.requiresBaseUrl && !data.baseUrl) throw new HttpError(422, 'validation', 'This provider type needs a server URL');
const at = now();
const key = app.config.secretKey;
if (id) {
const row = providerRow(app.db, id);
const apiKeyEnc = data.apiKey === undefined ? row.api_key_enc : data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null;
const headersEnc = data.headers === undefined ? row.headers_enc : Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null;
app.db.run(
'UPDATE ai_providers SET name = ?, kind = ?, base_url = ?, api_key_enc = ?, headers_enc = ?, enabled = ?, updated_at = ? WHERE id = ?',
data.name,
data.kind,
data.baseUrl,
apiKeyEnc,
headersEnc,
data.enabled === undefined ? row.enabled : data.enabled ? 1 : 0,
at,
id,
);
return mapProvider(app, providerRow(app.db, id));
}
const newProviderId = newId();
app.db.run(
'INSERT INTO ai_providers (id, name, kind, base_url, api_key_enc, headers_enc, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
newProviderId,
data.name,
data.kind,
data.baseUrl,
data.apiKey ? encryptSecret(data.apiKey.trim(), key) : null,
data.headers && Object.keys(data.headers).length ? encryptSecret(JSON.stringify(data.headers), key) : null,
data.enabled === false ? 0 : 1,
at,
at,
);
return mapProvider(app, providerRow(app.db, newProviderId));
}
export function deleteProvider(app: AppContext, id: string): void {
providerRow(app.db, id);
const modelIds = new Set(app.db.all<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ?', id).map((m) => m.id));
app.db.run('DELETE FROM ai_providers WHERE id = ?', id);
const tasks = getTaskSettings(app.db);
let changed = false;
for (const name of Object.keys(tasks) as (keyof TaskSettings)[]) {
if (tasks[name].modelId && modelIds.has(tasks[name].modelId!)) {
tasks[name].modelId = null;
changed = true;
}
}
if (changed) setSetting(app.db, 'ai.tasks', tasks);
}
export async function testProvider(app: AppContext, id: string) {
const row = providerRow(app.db, id);
const anyModel = app.db.get<{ model: string }>('SELECT model FROM ai_models WHERE provider_id = ? ORDER BY enabled DESC LIMIT 1', id)?.model;
const result = await getAdapter(row.kind).test(providerConfig(app, row), anyModel);
app.db.run('UPDATE ai_providers SET last_test_at = ?, last_test_ok = ?, last_test_message = ? WHERE id = ?', now(), result.ok ? 1 : 0, result.message.slice(0, 500), id);
return result;
}
/** Looks up the provider's model list and merges it into ai_models without disturbing user edits. */
export async function discoverModels(app: AppContext, id: string): Promise<{ added: number; total: number }> {
const row = providerRow(app.db, id);
const found = await getAdapter(row.kind).listModels(providerConfig(app, row));
const existing = new Map(app.db.all<ModelRow>('SELECT * FROM ai_models WHERE provider_id = ?', id).map((m) => [m.model, m]));
const at = now();
let added = 0;
app.db.tx(() => {
for (const m of found) {
const current = existing.get(m.id);
if (current) {
// Refresh facts the provider reports, but never overwrite prices or flags the user may have edited.
app.db.run(
'UPDATE ai_models SET context_window = COALESCE(?, context_window), max_output = COALESCE(?, max_output), price_in = COALESCE(price_in, ?), price_out = COALESCE(price_out, ?), updated_at = ? WHERE id = ?',
m.contextWindow ?? null,
m.maxOutput ?? null,
m.priceIn ?? null,
m.priceOut ?? null,
at,
current.id,
);
continue;
}
insertModel(app.db, id, m, 'discovered', false);
added += 1;
}
});
return { added, total: found.length };
}
function insertModel(db: DB, providerId: string, m: DiscoveredModel, source: string, enabled: boolean): string {
const id = newId();
const at = now();
db.run(
`INSERT INTO ai_models (id, provider_id, model, label, source, enabled, capabilities, context_window, max_output, price_in, price_out, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
id,
providerId,
m.id,
m.label ?? '',
source,
enabled ? 1 : 0,
JSON.stringify(m.capabilities ?? {}),
m.contextWindow ?? null,
m.maxOutput ?? null,
m.priceIn ?? null,
m.priceOut ?? null,
at,
at,
);
return id;
}
/* ------------------------------ models ------------------------------ */
export function listModels(db: DB): ModelDto[] {
return db.all<ModelRow>(`${MODEL_SELECT} ORDER BY p.created_at, m.enabled DESC, m.model`).map(mapModel);
}
export function getModel(db: DB, id: string): ModelDto {
const row = db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ?`, id);
if (!row) throw notFound('Model');
return mapModel(row);
}
export const manualModelSchema = z.object({
providerId: z.string(),
model: z.string().trim().min(1).max(200),
label: z.string().trim().max(120).optional(),
});
export function addManualModel(app: AppContext, input: z.input<typeof manualModelSchema>): ModelDto {
const data = manualModelSchema.parse(input);
const provider = providerRow(app.db, data.providerId);
const existing = app.db.get<{ id: string }>('SELECT id FROM ai_models WHERE provider_id = ? AND model = ?', provider.id, data.model);
if (existing) {
app.db.run('UPDATE ai_models SET enabled = 1, updated_at = ? WHERE id = ?', now(), existing.id);
return getModel(app.db, existing.id);
}
const defaults = lookupModelDefaults(provider.kind, data.model);
const id = insertModel(app.db, provider.id, { ...defaults, id: data.model, label: data.label || defaults.label }, 'manual', true);
return getModel(app.db, id);
}
export const modelPatchSchema = z.object({
enabled: z.boolean().optional(),
label: z.string().trim().max(120).optional(),
capabilities: z
.object({ tools: z.boolean(), vision: z.boolean(), pdf: z.boolean(), reasoning: z.boolean(), temperature: z.boolean() })
.partial()
.optional(),
contextWindow: z.number().int().min(256).max(100_000_000).nullable().optional(),
maxOutput: z.number().int().min(16).max(10_000_000).nullable().optional(),
priceIn: z.number().min(0).max(100000).nullable().optional(),
priceOut: z.number().min(0).max(100000).nullable().optional(),
});
export function patchModel(db: DB, id: string, input: z.input<typeof modelPatchSchema>): ModelDto {
const patch = modelPatchSchema.parse(input);
const row = db.get<ModelRow>('SELECT * FROM ai_models WHERE id = ?', id);
if (!row) throw notFound('Model');
const caps = { ...parseJson<Partial<ModelCapabilities>>(row.capabilities, {}), ...patch.capabilities };
db.run(
'UPDATE ai_models SET enabled = ?, label = ?, capabilities = ?, context_window = ?, max_output = ?, price_in = ?, price_out = ?, updated_at = ? WHERE id = ?',
patch.enabled === undefined ? row.enabled : patch.enabled ? 1 : 0,
patch.label ?? row.label,
JSON.stringify(caps),
patch.contextWindow === undefined ? row.context_window : patch.contextWindow,
patch.maxOutput === undefined ? row.max_output : patch.maxOutput,
patch.priceIn === undefined ? row.price_in : patch.priceIn,
patch.priceOut === undefined ? row.price_out : patch.priceOut,
now(),
id,
);
return getModel(db, id);
}
export function deleteModel(db: DB, id: string): void {
const { changes } = db.run('DELETE FROM ai_models WHERE id = ?', id);
if (!changes) throw notFound('Model');
}
/* ------------------------------ task routing ------------------------------ */
export function getTaskSettings(db: DB): TaskSettings {
const stored = getSetting<Partial<TaskSettings>>(db, 'ai.tasks', {});
const out = { default: { ...EMPTY_TASK, ...stored.default } } as TaskSettings;
for (const t of TASKS) out[t] = { ...EMPTY_TASK, ...stored[t] };
return out;
}
export function saveTaskSettings(db: DB, input: unknown): TaskSettings {
const schema = z.object({ default: taskConfigSchema, chat: taskConfigSchema, generate: taskConfigSchema, vision: taskConfigSchema, quick: taskConfigSchema }).partial();
const patch = schema.parse(input);
const next = { ...getTaskSettings(db), ...patch } as TaskSettings;
setSetting(db, 'ai.tasks', next);
return next;
}
export interface ResolvedModel {
provider: ProviderConfig;
model: ModelDto;
params: { temperature?: number; reasoningEffort?: ReasoningEffort; maxOutputTokens?: number };
}
/**
* Picks the model for a task: explicit override, then the task's own model, then the
* default model, then (so a fresh install works after adding one model) any enabled model.
*/
export function resolveModel(app: AppContext, task: TaskName, overrideModelId?: string | null): ResolvedModel {
const tasks = getTaskSettings(app.db);
const cfg = tasks[task];
const candidates = [overrideModelId, cfg.modelId, tasks.default.modelId].filter((x): x is string => Boolean(x));
let model: ModelDto | undefined;
for (const id of candidates) {
const row = app.db.get<ModelRow>(`${MODEL_SELECT} WHERE m.id = ? AND p.enabled = 1`, id);
if (row) {
model = mapModel(row);
break;
}
}
if (!model) {
const needVision = task === 'vision';
const rows = app.db.all<ModelRow>(`${MODEL_SELECT} WHERE m.enabled = 1 AND p.enabled = 1 ORDER BY p.created_at, m.created_at`).map(mapModel);
model = needVision ? rows.find((m) => m.capabilities.vision) : rows[0];
}
if (!model) {
throw new AIError(
'not_configured',
task === 'vision' ? 'No vision-capable AI model is set up. Add one in Settings, then choose it for "Vision and OCR".' : 'No AI model is set up yet. Add a provider and enable a model in Settings.',
);
}
const provider = providerConfig(app, providerRow(app.db, model.providerId));
const own = cfg.modelId === model.id || (!cfg.modelId && tasks.default.modelId === model.id) ? cfg : EMPTY_TASK;
const fallback = tasks.default;
const params: ResolvedModel['params'] = {};
const temperature = own.temperature ?? fallback.temperature;
const effort = own.reasoningEffort ?? fallback.reasoningEffort;
const maxOut = own.maxOutputTokens ?? fallback.maxOutputTokens;
if (temperature !== null && model.capabilities.temperature) params.temperature = temperature;
if (effort !== null && model.capabilities.reasoning) params.reasoningEffort = effort;
if (maxOut !== null) params.maxOutputTokens = maxOut;
return { provider, model, params };
}
/* ------------------------------ usage ------------------------------ */
export function estimateCost(model: Pick<ModelDto, 'priceIn' | 'priceOut'>, usage: Usage): number | null {
if (model.priceIn === null && model.priceOut === null) return null;
// Cached input is billed at a fraction of the list price by every provider; 10% is a close, conservative stand-in.
const cached = usage.cacheReadTokens ?? 0;
const fresh = Math.max(0, usage.inputTokens - cached);
return ((fresh + cached * 0.1) * (model.priceIn ?? 0) + usage.outputTokens * (model.priceOut ?? 0)) / 1_000_000;
}
function recordUsage(app: AppContext, r: ResolvedModel, task: string, usage: Usage | null, started: number, error?: string): void {
try {
app.db.run(
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
now(),
r.provider.id,
r.provider.name,
r.model.model,
task,
usage?.inputTokens ?? 0,
usage?.outputTokens ?? 0,
usage?.cacheReadTokens ?? 0,
usage ? estimateCost(r.model, usage) : null,
error ? 0 : 1,
(error ?? '').slice(0, 300),
now() - started,
);
} catch {
/* accounting must never break a request */
}
}
/* ------------------------------ call primitives ------------------------------ */
export interface ChatOptions {
task: TaskName;
/** Label stored with usage rows ("assistant", "quiz generation"...). Defaults to the task. */
purpose?: string;
modelId?: string | null;
system?: string;
messages: ChatMessage[];
tools?: ToolDef[];
toolChoice?: ChatRequest['toolChoice'];
jsonMode?: boolean;
temperature?: number;
reasoningEffort?: ReasoningEffort;
maxOutputTokens?: number;
signal?: AbortSignal;
onEvent?: (event: StreamEvent) => void;
}
export interface ChatResult {
message: ChatMessage;
text: string;
stopReason: StopReason;
usage: Usage;
model: ModelDto;
}
const sleep = (ms: number, signal?: AbortSignal) =>
new Promise<void>((resolve, reject) => {
const t = setTimeout(resolve, ms);
signal?.addEventListener('abort', () => {
clearTimeout(t);
reject(new AIError('aborted', 'The request was cancelled.'));
});
});
/** Runs one model turn. Retries transient failures, but only while nothing has been streamed yet. */
export async function runChat(app: AppContext, opts: ChatOptions): Promise<ChatResult> {
const resolved = resolveModel(app, opts.task, opts.modelId);
const caps = resolved.model.capabilities;
const req: ChatRequest = {
model: resolved.model.model,
system: opts.system,
messages: opts.messages,
tools: opts.tools,
toolChoice: opts.toolChoice,
jsonMode: opts.jsonMode,
capabilities: caps,
signal: opts.signal,
temperature: opts.temperature ?? resolved.params.temperature,
reasoningEffort: opts.reasoningEffort ?? resolved.params.reasoningEffort,
maxOutputTokens: opts.maxOutputTokens ?? resolved.params.maxOutputTokens ?? undefined,
};
if (req.maxOutputTokens && resolved.model.maxOutput) req.maxOutputTokens = Math.min(req.maxOutputTokens, resolved.model.maxOutput);
if (opts.tools?.length && !caps.tools && (opts.toolChoice === 'required' || typeof opts.toolChoice === 'object')) {
throw new AIError('unsupported', `${resolved.model.label} is marked as not supporting tools. Choose another model for this task.`);
}
const adapter = getAdapter(resolved.provider.kind);
const purpose = opts.purpose ?? opts.task;
for (let attempt = 0; ; attempt += 1) {
const started = now();
let streamed = false;
try {
for await (const event of adapter.stream(resolved.provider, req)) {
if (event.type === 'done') {
recordUsage(app, resolved, purpose, event.usage, started);
const text = event.message.content.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text).join('');
return { message: event.message, text, stopReason: event.stopReason, usage: event.usage, model: resolved.model };
}
streamed = true;
opts.onEvent?.(event);
}
throw new AIError('invalid_response', `${resolved.provider.name} ended the response unexpectedly.`, { retryable: true });
} catch (raw) {
const err = raw instanceof AIError ? raw : new AIError('server', 'The AI request failed unexpectedly.', { cause: raw });
if (!(raw instanceof AIError)) app.log('error', 'Unexpected AI adapter error', raw);
recordUsage(app, resolved, purpose, null, started, `${err.code}: ${err.message}`);
if (!err.retryable || streamed || attempt >= 2 || opts.signal?.aborted) throw err;
await sleep(1500 * (attempt + 1) ** 2, opts.signal);
}
}
}
/** Finds the first JSON value in model text (handles code fences and stray prose). */
export function parseLooseJson(text: string): unknown {
let t = text.trim();
const fence = /```(?:json)?\s*\n?([\s\S]*?)```/i.exec(t);
if (fence) t = fence[1].trim();
const start = t.search(/[[{]/);
if (start < 0) throw new AIError('invalid_response', 'The model did not return structured data.');
t = t.slice(start);
try {
return JSON.parse(t);
} catch {
try {
return JSON.parse(jsonrepair(t));
} catch {
throw new AIError('invalid_response', 'The model returned data that could not be read.', { retryable: true });
}
}
}
export interface StructuredOptions extends Omit<ChatOptions, 'tools' | 'toolChoice' | 'jsonMode'> {
/** Name + description of the "function" whose arguments are the result. */
toolName: string;
toolDescription: string;
schema: Record<string, unknown>;
}
/**
* Structured output that works on every provider: the schema is offered as a single tool the
* model must call. Models without tool support (many local ones) get the schema in the prompt
* and their text is parsed leniently instead.
*/
export async function generateStructured(app: AppContext, opts: StructuredOptions): Promise<{ data: unknown; usage: Usage; model: ModelDto; truncated: boolean }> {
const resolved = resolveModel(app, opts.task, opts.modelId);
const tool: ToolDef = { name: opts.toolName, description: opts.toolDescription, parameters: opts.schema };
const promptJson = async () => {
const system = `${opts.system ?? ''}\n\nRespond with ONLY a JSON object matching this JSON Schema. No prose, no Markdown fences.\n${JSON.stringify(opts.schema)}`.trim();
const res = await runChat(app, { ...opts, system, jsonMode: true });
return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
};
if (!resolved.model.capabilities.tools) return promptJson();
try {
const res = await runChat(app, { ...opts, tools: [tool], toolChoice: { name: opts.toolName } });
const call = res.message.content.find((p) => p.type === 'tool_call');
if (call && call.type === 'tool_call') return { data: call.input, usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.text.trim()) return { data: parseLooseJson(res.text), usage: res.usage, model: res.model, truncated: res.stopReason === 'max_tokens' };
if (res.stopReason === 'refusal') throw new AIError('content_filter', 'The model declined this request.');
if (res.stopReason === 'max_tokens') throw new AIError('invalid_response', 'The model ran out of output space before finishing. Ask for fewer items at a time, or raise the output limit in Settings.');
throw new AIError('invalid_response', 'The model returned nothing usable.', { retryable: true });
} catch (err) {
if (err instanceof AIError && err.code === 'unsupported') return promptJson();
throw err;
}
}
/* ------------------------------ reporting ------------------------------ */
export function usageSummary(db: DB, days: number) {
const since = now() - days * 86_400_000;
const byModel = db.all<Record<string, any>>(
`SELECT provider_name AS provider, model, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost,
SUM(CASE WHEN ok = 0 THEN 1 ELSE 0 END) AS errors, SUM(CASE WHEN cost IS NULL AND ok = 1 THEN 1 ELSE 0 END) AS unpriced
FROM ai_usage WHERE at >= ? GROUP BY provider_name, model ORDER BY cost DESC, calls DESC`,
since,
);
const byTask = db.all<Record<string, any>>(
`SELECT task, COUNT(*) AS calls, SUM(input_tokens) AS input, SUM(output_tokens) AS output, SUM(cost) AS cost FROM ai_usage WHERE at >= ? AND ok = 1 GROUP BY task ORDER BY cost DESC`,
since,
);
const total = db.get<Record<string, any>>(
`SELECT COUNT(*) AS calls, COALESCE(SUM(input_tokens), 0) AS input, COALESCE(SUM(output_tokens), 0) AS output, COALESCE(SUM(cost), 0) AS cost FROM ai_usage WHERE at >= ?`,
since,
)!;
const recentErrors = db.all<Record<string, any>>(`SELECT at, provider_name AS provider, model, task, error FROM ai_usage WHERE ok = 0 AND at >= ? ORDER BY at DESC LIMIT 10`, since);
return { days, total, byModel, byTask, recentErrors };
}
/** True when at least one usable model exists; lets the UI show setup prompts instead of errors. */
export function aiStatus(app: AppContext) {
const models = listModels(app.db).filter((m) => m.available);
const tasks = getTaskSettings(app.db);
const resolved: Record<string, { modelId: string; label: string; provider: string } | null> = {};
for (const t of TASKS) {
try {
const r = resolveModel(app, t);
resolved[t] = { modelId: r.model.id, label: r.model.label, provider: r.provider.name };
} catch {
resolved[t] = null;
}
}
return { ready: models.length > 0, modelCount: models.length, tasks, resolved };
}

180
server/ai/types.ts Normal file
View file

@ -0,0 +1,180 @@
/**
* Provider-neutral AI contract.
*
* Every provider adapter (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama)
* translates between this shape and the provider's wire format. Nothing outside
* `server/ai/providers/` may depend on a provider-specific request or response.
*/
export type ProviderKind = 'openai' | 'anthropic' | 'gemini' | 'openai_compatible' | 'ollama';
export const PROVIDER_KINDS: ProviderKind[] = ['openai', 'anthropic', 'gemini', 'openai_compatible', 'ollama'];
/** Decrypted provider connection, only ever held server-side. */
export interface ProviderConfig {
id: string;
kind: ProviderKind;
name: string;
/** Null means "use the provider's official default endpoint". */
baseUrl: string | null;
apiKey: string | null;
/** Extra HTTP headers (e.g. OpenRouter attribution, org ids). */
headers?: Record<string, string>;
}
export type ContentPart =
| { type: 'text'; text: string }
/** Base64 image data (no data: prefix). */
| { type: 'image'; mime: string; data: string }
/** Base64 file, used for native PDF input where a provider supports it. */
| { type: 'file'; mime: string; data: string; filename?: string }
| { type: 'tool_call'; id: string; name: string; input: Record<string, unknown> }
| { type: 'tool_result'; toolCallId: string; name: string; output: string; isError?: boolean };
export interface ChatMessage {
/** Tool results travel as `tool_result` parts inside a `user` message. */
role: 'user' | 'assistant';
content: ContentPart[];
/**
* Opaque provider state for an assistant turn (thinking blocks + signatures,
* Gemini thought signatures, OpenAI reasoning items...). An adapter writes it
* on the `done` event and, when the next request goes to the same provider
* kind, replays it verbatim instead of rebuilding the turn from `content`.
*/
providerData?: { kind: ProviderKind; data: unknown };
}
export interface ToolDef {
name: string;
description: string;
/** JSON Schema (object). Adapters down-convert for providers with a restricted dialect. */
parameters: Record<string, unknown>;
}
export type ReasoningEffort = 'none' | 'low' | 'medium' | 'high';
export interface ModelCapabilities {
tools: boolean;
vision: boolean;
/** Accepts PDF files directly as input. */
pdf: boolean;
/** Has a configurable reasoning / thinking effort. */
reasoning: boolean;
/** Accepts a custom temperature (some reasoning models reject it). */
temperature: boolean;
}
export interface ChatRequest {
model: string;
system?: string;
messages: ChatMessage[];
tools?: ToolDef[];
toolChoice?: 'auto' | 'none' | 'required' | { name: string };
temperature?: number;
maxOutputTokens?: number;
reasoningEffort?: ReasoningEffort;
/** Ask for a JSON object as the text output when the provider has a JSON mode. */
jsonMode?: boolean;
/** Known capabilities of the model, when the caller has them. Adapters must not send unsupported params. */
capabilities?: Partial<ModelCapabilities>;
signal?: AbortSignal;
}
export interface Usage {
inputTokens: number;
outputTokens: number;
cacheReadTokens?: number;
cacheWriteTokens?: number;
reasoningTokens?: number;
}
export type StopReason = 'end' | 'tool_use' | 'max_tokens' | 'refusal' | 'other';
export type StreamEvent =
| { type: 'text'; delta: string }
| { type: 'reasoning'; delta: string }
/** Emitted once per tool call, after its arguments are complete and parsed. */
| { type: 'tool_call'; call: { id: string; name: string; input: Record<string, unknown> } }
/** Always the final event of a successful stream. */
| { type: 'done'; message: ChatMessage; stopReason: StopReason; usage: Usage };
export interface DiscoveredModel {
/** Identifier sent to the API. */
id: string;
label?: string;
contextWindow?: number;
maxOutput?: number;
capabilities?: Partial<ModelCapabilities>;
/** USD per million tokens, when known. */
priceIn?: number;
priceOut?: number;
}
export interface TestResult {
ok: boolean;
message: string;
latencyMs: number;
/** Number of models visible to the key, when discovery works. */
modelCount?: number;
}
export interface ProviderAdapter {
kind: ProviderKind;
/** Human label and defaults used by the settings UI. */
meta: {
label: string;
defaultBaseUrl: string | null;
requiresApiKey: boolean;
requiresBaseUrl: boolean;
supportsDiscovery: boolean;
docsUrl: string;
keyHint: string;
};
/** Streams one model turn. Throws AIError; never yields after `done`. */
stream(cfg: ProviderConfig, req: ChatRequest): AsyncIterable<StreamEvent>;
/** Lists chat-capable models. Throws AIError('unsupported') when a provider has no listing API. */
listModels(cfg: ProviderConfig, signal?: AbortSignal): Promise<DiscoveredModel[]>;
/** Cheap credential / connectivity check. Never throws. */
test(cfg: ProviderConfig, model?: string): Promise<TestResult>;
}
export type AIErrorCode =
| 'auth'
| 'rate_limit'
| 'quota'
| 'context_length'
| 'not_found'
| 'bad_request'
| 'content_filter'
| 'server'
| 'network'
| 'timeout'
| 'aborted'
| 'unsupported'
| 'invalid_response'
| 'not_configured';
export class AIError extends Error {
code: AIErrorCode;
status?: number;
retryable: boolean;
/** Raw (already secret-scrubbed) provider message, for the details disclosure in the UI. */
providerMessage?: string;
constructor(code: AIErrorCode, message: string, opts: { status?: number; retryable?: boolean; providerMessage?: string; cause?: unknown } = {}) {
super(message, opts.cause !== undefined ? { cause: opts.cause } : undefined);
this.name = 'AIError';
this.code = code;
this.status = opts.status;
this.retryable = opts.retryable ?? (code === 'rate_limit' || code === 'server' || code === 'network' || code === 'timeout');
this.providerMessage = opts.providerMessage;
}
}
export const DEFAULT_CAPABILITIES: ModelCapabilities = {
tools: true,
vision: false,
pdf: false,
reasoning: false,
temperature: true,
};

277
server/auth.ts Normal file
View file

@ -0,0 +1,277 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { Hono, type Context, type MiddlewareHandler } from 'hono';
import { deleteCookie, getCookie, setCookie } from 'hono/cookie';
import { getConnInfo } from '@hono/node-server/conninfo';
import { z } from 'zod';
import { hashPassword, safeEqual, verifyPassword } from './crypto.ts';
import { HttpError, readBody, type AppContext } from './http.ts';
import { deleteSetting, getSetting, setSetting } from './settings.ts';
import { newToken, now, sha256 } from './util.ts';
const COOKIE = 'sd_session';
const SESSION_MS = 30 * 86_400_000;
const PASSWORD_MIN = 8;
const passwordSchema = z.string().min(PASSWORD_MIN, `Use at least ${PASSWORD_MIN} characters`).max(256);
/* ------------------------------ rate limiting ------------------------------ */
interface Bucket {
failures: number[];
}
const WINDOW_MS = 15 * 60_000;
const MAX_PER_IP = 8;
const MAX_GLOBAL = 40;
const buckets = new Map<string, Bucket>();
function recent(key: string, at: number): number[] {
const bucket = buckets.get(key) ?? { failures: [] };
bucket.failures = bucket.failures.filter((t) => at - t < WINDOW_MS);
buckets.set(key, bucket);
return bucket.failures;
}
function assertNotLimited(ip: string): void {
const at = now();
const mine = recent(`ip:${ip}`, at);
const all = recent('global', at);
if (mine.length >= MAX_PER_IP || all.length >= MAX_GLOBAL) {
const oldest = (mine.length >= MAX_PER_IP ? mine : all)[0];
const retry = Math.max(1, Math.ceil((oldest + WINDOW_MS - at) / 1000));
throw new HttpError(429, 'rate_limited', `Too many attempts. Try again in ${Math.ceil(retry / 60)} minute${retry > 60 ? 's' : ''}.`, { retryAfter: retry });
}
}
function recordFailure(ip: string): void {
const at = now();
recent(`ip:${ip}`, at).push(at);
recent('global', at).push(at);
}
export function resetRateLimits(): void {
buckets.clear();
}
/* ------------------------------ request helpers ------------------------------ */
export function clientIp(c: Context, trustProxy: boolean): string {
if (trustProxy) {
const fwd = c.req.header('x-forwarded-for');
if (fwd) return fwd.split(',')[0].trim();
const real = c.req.header('x-real-ip');
if (real) return real.trim();
}
try {
return getConnInfo(c).remote.address ?? 'unknown';
} catch {
return 'unknown';
}
}
export function isHttps(c: Context, trustProxy: boolean): boolean {
if (trustProxy) {
const proto = c.req.header('x-forwarded-proto');
if (proto) return proto.split(',')[0].trim() === 'https';
}
return new URL(c.req.url).protocol === 'https:';
}
/* ------------------------------ sessions ------------------------------ */
function createSession(app: AppContext, c: Context): void {
const token = newToken();
const at = now();
app.db.run(
'INSERT INTO sessions (token_hash, created_at, expires_at, last_seen_at, user_agent, ip) VALUES (?, ?, ?, ?, ?, ?)',
sha256(token),
at,
at + SESSION_MS,
at,
(c.req.header('user-agent') ?? '').slice(0, 300),
clientIp(c, app.config.trustProxy),
);
app.db.run('DELETE FROM sessions WHERE expires_at < ?', at);
const secure = app.config.cookieSecure === 'auto' ? isHttps(c, app.config.trustProxy) : app.config.cookieSecure;
setCookie(c, COOKIE, token, { httpOnly: true, sameSite: 'Lax', secure, path: '/', maxAge: Math.floor(SESSION_MS / 1000) });
}
export function isAuthenticated(app: AppContext, c: Context): boolean {
const token = getCookie(c, COOKIE);
if (!token) return false;
const hash = sha256(token);
const at = now();
const row = app.db.get<{ expires_at: number; last_seen_at: number }>('SELECT expires_at, last_seen_at FROM sessions WHERE token_hash = ?', hash);
if (!row || row.expires_at < at) return false;
// Sliding expiry, written at most once an hour.
if (at - row.last_seen_at > 3_600_000) {
app.db.run('UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE token_hash = ?', at, at + SESSION_MS, hash);
}
return true;
}
export function requireAuth(app: AppContext): MiddlewareHandler {
return async (c, next) => {
if (!isAuthenticated(app, c)) throw new HttpError(401, 'unauthorized', 'Sign in to continue');
await next();
};
}
/**
* CSRF defence for cookie-authenticated requests: state-changing calls must carry a custom
* header (which a cross-site form cannot send, and which forces a CORS preflight that this
* server never approves), and must not be flagged cross-site by the browser.
*/
export function csrfGuard(): MiddlewareHandler {
return async (c, next) => {
const method = c.req.method;
if (method !== 'GET' && method !== 'HEAD' && method !== 'OPTIONS') {
if (c.req.header('x-requested-with') !== 'studydesk') throw new HttpError(403, 'csrf', 'Missing request header');
// Fetch metadata is set by the browser and cannot be forged by page script. Using it
// (rather than comparing Origin to Host) keeps this working behind proxies that rewrite Host.
if (c.req.header('sec-fetch-site') === 'cross-site') throw new HttpError(403, 'csrf', 'Cross-site request blocked');
}
await next();
};
}
/* ------------------------------ password storage ------------------------------ */
const HASH_KEY = 'auth.password_hash';
const SETUP_KEY = 'auth.setup_code';
export function isConfigured(app: AppContext): boolean {
return Boolean(getSetting<string | null>(app.db, HASH_KEY, null));
}
function setupCodeFile(app: AppContext): string {
return path.join(app.config.dataDir, 'setup-code.txt');
}
async function storePassword(app: AppContext, password: string): Promise<void> {
setSetting(app.db, HASH_KEY, await hashPassword(password));
deleteSetting(app.db, SETUP_KEY);
fs.rmSync(setupCodeFile(app), { force: true });
}
/**
* Runs at startup. Applies ADMIN_PASSWORD / RESET_ADMIN_PASSWORD, or prepares a one-time
* setup code (printed to the log and written to the data folder) so that only someone with
* access to the server can claim a fresh install.
*/
export async function initAuth(app: AppContext): Promise<void> {
const { config, db } = app;
if (config.resetAdminPassword) {
passwordSchema.parse(config.resetAdminPassword);
await storePassword(app, config.resetAdminPassword);
db.run('DELETE FROM sessions');
app.log('warn', 'Administrator password was reset from RESET_ADMIN_PASSWORD. Remove that variable and restart.');
return;
}
if (isConfigured(app)) return;
if (config.adminPassword) {
passwordSchema.parse(config.adminPassword);
await storePassword(app, config.adminPassword);
app.log('info', 'Administrator password initialised from ADMIN_PASSWORD.');
return;
}
let code = getSetting<string | null>(db, SETUP_KEY, null);
if (!code) {
code = crypto.randomBytes(5).toString('hex').toUpperCase().replace(/(.{5})/, '$1-');
setSetting(db, SETUP_KEY, code);
}
try {
fs.writeFileSync(setupCodeFile(app), `${code}\n`, { mode: 0o600 });
} catch {
/* the log line below is enough */
}
app.log('info', `First-run setup code: ${code} (also saved to ${setupCodeFile(app)})`);
}
/* ------------------------------ routes ------------------------------ */
export function authRoutes(app: AppContext): Hono {
const r = new Hono();
r.get('/status', (c) => c.json({ configured: isConfigured(app), authenticated: isAuthenticated(app, c) }));
r.post('/setup', async (c) => {
if (isConfigured(app)) throw new HttpError(409, 'already_configured', 'Study Desk is already set up');
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ code: z.string().max(40), password: passwordSchema }));
const expected = getSetting<string | null>(app.db, SETUP_KEY, null);
const given = body.code.trim().toUpperCase();
if (!expected || !(safeEqual(given, expected) || safeEqual(given, expected.replace('-', '')))) {
recordFailure(ip);
throw new HttpError(403, 'bad_setup_code', 'That setup code is not correct. Check the server log or setup-code.txt in the data folder.');
}
await storePassword(app, body.password);
createSession(app, c);
return c.json({ ok: true });
});
r.post('/login', async (c) => {
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ password: z.string().max(256) }));
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
if (!hash) throw new HttpError(409, 'not_configured', 'Study Desk has not been set up yet');
if (!(await verifyPassword(body.password, hash))) {
recordFailure(ip);
throw new HttpError(401, 'bad_password', 'Incorrect password');
}
createSession(app, c);
return c.json({ ok: true });
});
r.post('/logout', (c) => {
const token = getCookie(c, COOKIE);
if (token) app.db.run('DELETE FROM sessions WHERE token_hash = ?', sha256(token));
deleteCookie(c, COOKIE, { path: '/' });
return c.json({ ok: true });
});
r.post('/password', requireAuth(app), async (c) => {
const ip = clientIp(c, app.config.trustProxy);
assertNotLimited(ip);
const body = await readBody(c, z.object({ current: z.string().max(256), next: passwordSchema }));
const hash = getSetting<string | null>(app.db, HASH_KEY, null);
if (!hash || !(await verifyPassword(body.current, hash))) {
recordFailure(ip);
throw new HttpError(403, 'bad_password', 'Current password is incorrect');
}
await storePassword(app, body.next);
// Sign out every other device.
const token = getCookie(c, COOKIE);
app.db.run('DELETE FROM sessions WHERE token_hash != ?', token ? sha256(token) : '');
return c.json({ ok: true });
});
r.get('/sessions', requireAuth(app), (c) => {
const current = sha256(getCookie(c, COOKIE) ?? '');
const rows = app.db.all<{ token_hash: string; created_at: number; last_seen_at: number; user_agent: string; ip: string }>(
'SELECT token_hash, created_at, last_seen_at, user_agent, ip FROM sessions WHERE expires_at > ? ORDER BY last_seen_at DESC',
now(),
);
return c.json({
sessions: rows.map((s) => ({
id: s.token_hash.slice(0, 12),
createdAt: s.created_at,
lastSeenAt: s.last_seen_at,
userAgent: s.user_agent,
ip: s.ip,
current: s.token_hash === current,
})),
});
});
r.post('/sessions/revoke-others', requireAuth(app), (c) => {
const current = sha256(getCookie(c, COOKIE) ?? '');
const { changes } = app.db.run('DELETE FROM sessions WHERE token_hash != ?', current);
return c.json({ revoked: changes });
});
return r;
}

89
server/config.ts Normal file
View file

@ -0,0 +1,89 @@
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
export interface Config {
env: 'development' | 'production' | 'test';
host: string;
port: number;
dataDir: string;
dbPath: string;
uploadsDir: string;
mediaDir: string;
backupsDir: string;
/** Directory with the built web app, or null when the Vite dev server serves it. */
webDir: string | null;
/** 32-byte key used to encrypt provider API keys at rest. */
secretKey: Buffer;
/** 'auto' marks the cookie Secure whenever the request arrived over HTTPS (directly or via proxy). */
cookieSecure: 'auto' | boolean;
trustProxy: boolean;
adminPassword: string | null;
resetAdminPassword: string | null;
maxUploadBytes: number;
backupKeep: number;
timezone: string;
}
function bool(v: string | undefined, fallback: boolean): boolean {
if (v === undefined || v === '') return fallback;
return ['1', 'true', 'yes', 'on'].includes(v.toLowerCase());
}
function loadSecret(dataDir: string, fromEnv: string | undefined): Buffer {
if (fromEnv && fromEnv.trim().length >= 16) {
return crypto.createHash('sha256').update(`studydesk:${fromEnv.trim()}`).digest();
}
const file = path.join(dataDir, 'secret.key');
if (fs.existsSync(file)) {
const raw = Buffer.from(fs.readFileSync(file, 'utf8').trim(), 'base64');
if (raw.length === 32) return raw;
}
const key = crypto.randomBytes(32);
fs.writeFileSync(file, key.toString('base64'), { mode: 0o600 });
return key;
}
export function loadConfig(overrides: Partial<Config> = {}, envVars: NodeJS.ProcessEnv = process.env): Config {
const env = (overrides.env ?? (envVars.NODE_ENV === 'production' ? 'production' : envVars.NODE_ENV === 'test' ? 'test' : 'development')) as Config['env'];
const dataDir = path.resolve(overrides.dataDir ?? envVars.DATA_DIR ?? (env === 'production' ? '/data' : './data'));
const uploadsDir = path.join(dataDir, 'uploads');
const mediaDir = path.join(dataDir, 'media');
const backupsDir = path.join(dataDir, 'backups');
for (const dir of [dataDir, uploadsDir, mediaDir, backupsDir]) fs.mkdirSync(dir, { recursive: true });
const secure = (envVars.COOKIE_SECURE ?? 'auto').toLowerCase();
let webDir: string | null = overrides.webDir ?? envVars.WEB_DIR ?? null;
if (webDir === null) {
const candidate = path.resolve('dist/web');
if (fs.existsSync(path.join(candidate, 'index.html'))) webDir = candidate;
}
let timezone = envVars.TZ || Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC';
try {
new Intl.DateTimeFormat('en-US', { timeZone: timezone });
} catch {
timezone = 'UTC';
}
return {
env,
host: envVars.HOST ?? '0.0.0.0',
port: Number(envVars.PORT ?? 3726),
dataDir,
dbPath: path.join(dataDir, 'study.db'),
uploadsDir,
mediaDir,
backupsDir,
webDir,
secretKey: loadSecret(dataDir, envVars.APP_SECRET),
cookieSecure: secure === 'auto' ? 'auto' : bool(secure, false),
trustProxy: bool(envVars.TRUST_PROXY, true),
adminPassword: envVars.ADMIN_PASSWORD?.trim() || null,
resetAdminPassword: envVars.RESET_ADMIN_PASSWORD?.trim() || null,
maxUploadBytes: Math.max(1, Number(envVars.MAX_UPLOAD_MB ?? 100)) * 1024 * 1024,
backupKeep: Math.max(1, Number(envVars.BACKUP_KEEP ?? 7)),
timezone,
...overrides,
};
}

62
server/crypto.ts Normal file
View file

@ -0,0 +1,62 @@
import crypto from 'node:crypto';
const SCRYPT = { N: 32768, r: 8, p: 1, keylen: 32 };
function scrypt(password: string, salt: Buffer, N: number, r: number, p: number, keylen: number): Promise<Buffer> {
return new Promise((resolve, reject) => {
crypto.scrypt(password.normalize('NFKC'), salt, keylen, { N, r, p, maxmem: 256 * N * r }, (err, key) => (err ? reject(err) : resolve(key)));
});
}
/** scrypt password hash: `scrypt$N$r$p$salt$hash` (base64url fields). */
export async function hashPassword(password: string): Promise<string> {
const salt = crypto.randomBytes(16);
const key = await scrypt(password, salt, SCRYPT.N, SCRYPT.r, SCRYPT.p, SCRYPT.keylen);
return ['scrypt', SCRYPT.N, SCRYPT.r, SCRYPT.p, salt.toString('base64url'), key.toString('base64url')].join('$');
}
export async function verifyPassword(password: string, stored: string): Promise<boolean> {
const parts = stored.split('$');
if (parts.length !== 6 || parts[0] !== 'scrypt') return false;
const [, n, r, p, saltB64, hashB64] = parts;
const expected = Buffer.from(hashB64, 'base64url');
try {
const actual = await scrypt(password, Buffer.from(saltB64, 'base64url'), Number(n), Number(r), Number(p), expected.length);
return actual.length === expected.length && crypto.timingSafeEqual(actual, expected);
} catch {
return false;
}
}
/** AES-256-GCM. Output: `v1.<iv>.<tag>.<ciphertext>` (base64url). */
export function encryptSecret(plain: string, key: Buffer): string {
const iv = crypto.randomBytes(12);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const data = Buffer.concat([cipher.update(plain, 'utf8'), cipher.final()]);
return ['v1', iv.toString('base64url'), cipher.getAuthTag().toString('base64url'), data.toString('base64url')].join('.');
}
/** Returns null when the value cannot be decrypted (wrong key, tampering). */
export function decryptSecret(value: string | null | undefined, key: Buffer): string | null {
if (!value) return null;
const parts = value.split('.');
if (parts.length !== 4 || parts[0] !== 'v1') return null;
try {
const decipher = crypto.createDecipheriv('aes-256-gcm', key, Buffer.from(parts[1], 'base64url'));
decipher.setAuthTag(Buffer.from(parts[2], 'base64url'));
return Buffer.concat([decipher.update(Buffer.from(parts[3], 'base64url')), decipher.final()]).toString('utf8');
} catch {
return null;
}
}
export function maskSecret(secret: string | null): string {
if (!secret) return '';
return secret.length <= 8 ? '••••' : `${secret.slice(0, 3)}••••${secret.slice(-4)}`;
}
export function safeEqual(a: string, b: string): boolean {
const x = crypto.createHash('sha256').update(a).digest();
const y = crypto.createHash('sha256').update(b).digest();
return crypto.timingSafeEqual(x, y);
}

140
server/db.ts Normal file
View file

@ -0,0 +1,140 @@
import fs from 'node:fs';
import path from 'node:path';
import { DatabaseSync, type StatementSync } from 'node:sqlite';
import { MIGRATIONS } from './migrations.ts';
type Param = string | number | bigint | boolean | null | undefined | Uint8Array;
function bind(params: Param[]): (string | number | bigint | null | Uint8Array)[] {
return params.map((p) => (p === undefined ? null : typeof p === 'boolean' ? (p ? 1 : 0) : p));
}
/**
* Thin synchronous wrapper around node:sqlite. Single process, single writer:
* exactly what a one-user app in one container needs, with no native add-on to build.
*/
export class DB {
readonly raw: DatabaseSync;
readonly path: string;
private statements = new Map<string, StatementSync>();
private txDepth = 0;
constructor(file: string) {
this.path = file;
this.raw = new DatabaseSync(file);
this.raw.exec('PRAGMA journal_mode = WAL');
this.raw.exec('PRAGMA synchronous = NORMAL');
this.raw.exec('PRAGMA foreign_keys = ON');
this.raw.exec('PRAGMA busy_timeout = 5000');
}
private stmt(sql: string): StatementSync {
let s = this.statements.get(sql);
if (!s) {
s = this.raw.prepare(sql);
if (this.statements.size > 500) this.statements.clear();
this.statements.set(sql, s);
}
return s;
}
all<T = Record<string, unknown>>(sql: string, ...params: Param[]): T[] {
return this.stmt(sql).all(...bind(params)) as T[];
}
get<T = Record<string, unknown>>(sql: string, ...params: Param[]): T | undefined {
return this.stmt(sql).get(...bind(params)) as T | undefined;
}
run(sql: string, ...params: Param[]): { changes: number; lastId: number } {
const r = this.stmt(sql).run(...bind(params));
return { changes: Number(r.changes), lastId: Number(r.lastInsertRowid) };
}
exec(sql: string): void {
this.raw.exec(sql);
}
/** Runs fn atomically. Nested calls become savepoints. */
tx<T>(fn: () => T): T {
const depth = this.txDepth;
const name = `sp_${depth}`;
this.raw.exec(depth === 0 ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
this.txDepth += 1;
try {
const out = fn();
this.txDepth -= 1;
this.raw.exec(depth === 0 ? 'COMMIT' : `RELEASE ${name}`);
return out;
} catch (e) {
this.txDepth -= 1;
try {
this.raw.exec(depth === 0 ? 'ROLLBACK' : `ROLLBACK TO ${name}; RELEASE ${name}`);
} catch {
/* the original error matters more */
}
throw e;
}
}
/** Writes a consistent, compacted copy of the database (safe while the app is running). */
snapshot(target: string): void {
if (fs.existsSync(target)) fs.rmSync(target);
this.raw.exec(`VACUUM INTO '${target.replace(/'/g, "''")}'`);
}
close(): void {
this.statements.clear();
try {
this.raw.exec('PRAGMA wal_checkpoint(TRUNCATE)');
} catch {
/* ignore */
}
this.raw.close();
}
}
export interface MigrationReport {
applied: string[];
backup: string | null;
}
/**
* Applies pending migrations in order, each in its own transaction. When an existing
* database is about to change, a snapshot is written to the backups folder first.
*/
export function migrate(db: DB, backupsDir?: string): MigrationReport {
db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at INTEGER NOT NULL)');
const done = new Set(db.all<{ id: number }>('SELECT id FROM schema_migrations').map((r) => r.id));
const known = new Set(MIGRATIONS.map((m) => m.id));
const unknown = [...done].filter((id) => !known.has(id));
if (unknown.length > 0) {
throw new Error(
`This database was created by a newer version of Study Desk (unknown migration ${unknown.join(', ')}). Upgrade the app instead of downgrading it.`,
);
}
const pending = MIGRATIONS.filter((m) => !done.has(m.id)).sort((a, b) => a.id - b.id);
const report: MigrationReport = { applied: [], backup: null };
if (pending.length === 0) return report;
if (done.size > 0 && backupsDir && db.path !== ':memory:') {
fs.mkdirSync(backupsDir, { recursive: true });
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
report.backup = path.join(backupsDir, `pre-migration-${stamp}.db`);
db.snapshot(report.backup);
}
for (const m of pending) {
db.tx(() => {
db.exec(m.sql);
db.run('INSERT INTO schema_migrations (id, name, applied_at) VALUES (?, ?, ?)', m.id, m.name, Date.now());
});
report.applied.push(`${m.id}_${m.name}`);
}
return report;
}
export function openDatabase(file: string, backupsDir?: string): { db: DB; report: MigrationReport } {
const db = new DB(file);
const report = migrate(db, backupsDir);
return { db, report };
}

514
server/documents/chunk.ts Normal file
View file

@ -0,0 +1,514 @@
/**
* Text normalisation, token estimation and retrieval chunking.
* Pure functions, no dependencies. Everything here is linear-time on its input.
*/
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
// ---------------------------------------------------------------------------
// Token estimation
// ---------------------------------------------------------------------------
/**
* Un-rounded token estimate. Sub-additive under concatenation
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
*/
function rawTokens(text: string): number {
let total = 0;
let prevWhitespace = false;
for (let i = 0; i < text.length; i++) {
const c = text.charCodeAt(i);
if (c === 32 || (c >= 9 && c <= 13)) {
// A run of whitespace costs the same as one character of prose.
if (!prevWhitespace) total += 0.25;
prevWhitespace = true;
continue;
}
prevWhitespace = false;
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
else if (c >= 48 && c <= 57) total += 0.4;
else if (c < 128) total += 0.5;
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
else total += 0.5; // accented Latin, Greek, symbols
}
return total;
}
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
export function estimateTokens(text: string): number {
if (!text) return 0;
return Math.ceil(rawTokens(text));
}
// ---------------------------------------------------------------------------
// cleanText
// ---------------------------------------------------------------------------
const LIGATURES: Record<string, string> = {
'ff': 'ff',
'fi': 'fi',
'fl': 'fl',
'ffi': 'ffi',
'ffl': 'ffl',
'ſt': 'st',
'st': 'st',
};
/** First halves that normally form a real hyphenated compound ("well-known"). */
const HYPHEN_KEEP_LEFT = new Set([
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
]);
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
const HYPHEN_KEEP_RIGHT = new Set([
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
]);
function isLower(code: number): boolean {
return code >= 97 && code <= 122;
}
function isLetter(code: number): boolean {
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
}
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
function joinHyphenatedLineBreaks(text: string): string {
if (!text.includes('-\n')) return text;
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
let start = offset;
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
const left = whole.slice(start, offset);
if (left.length < 2) return match;
if (start > 0) {
const before = whole.charCodeAt(start - 1);
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
}
const rightStart = offset + match.length;
let rightEnd = rightStart;
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
const right = whole.slice(rightStart, rightEnd);
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
return '';
});
}
/**
* Normalises extracted text: newlines, unicode spaces, control characters,
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
* a line break. Idempotent.
*/
export function cleanText(text: string): string {
if (!text) return '';
let t = text;
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
t = t
.split('\n')
.map((line) => line.trimEnd())
.join('\n');
t = joinHyphenatedLineBreaks(t);
t = t.replace(/\n{3,}/g, '\n\n');
return t.trim();
}
// ---------------------------------------------------------------------------
// Repeated page headers / footers
// ---------------------------------------------------------------------------
function edgeLineKey(line: string): string | null {
const trimmed = line.trim();
if (!trimmed || trimmed.length > 100) return null;
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
}
/**
* Removes running headers/footers and page numbers from per-page text, but only
* when the same first or last line (digits ignored) shows up on at least 60% of
* the pages that have text, and there are at least four such pages. Chapter
* headings that change through a book are therefore left alone.
*/
export function stripRepeatedPageLines(pages: string[]): string[] {
let current = pages.map((page) => page.split('\n'));
for (let pass = 0; pass < 2; pass++) {
const firstIdx: number[] = [];
const lastIdx: number[] = [];
const topCounts = new Map<string, number>();
const bottomCounts = new Map<string, number>();
let textPages = 0;
for (let p = 0; p < current.length; p++) {
const lines = current[p];
let first = 0;
while (first < lines.length && !lines[first].trim()) first++;
let last = lines.length - 1;
while (last >= 0 && !lines[last].trim()) last--;
firstIdx.push(first < lines.length ? first : -1);
lastIdx.push(last);
if (last < 0) continue;
textPages++;
const top = edgeLineKey(lines[first]);
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
}
}
if (textPages < 4) break;
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
let removed = false;
current = current.map((lines, p) => {
const first = firstIdx[p];
const last = lastIdx[p];
if (last < 0) return lines;
let from = 0;
let to = lines.length;
const top = edgeLineKey(lines[first]);
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
}
if (from === 0 && to === lines.length) return lines;
removed = true;
return lines.slice(from, to);
});
if (!removed) break;
}
return current.map((lines) => lines.join('\n'));
}
// ---------------------------------------------------------------------------
// Chunking
// ---------------------------------------------------------------------------
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
interface Atom {
text: string;
raw: number;
sep: string;
sepRaw: number;
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
strength: number;
}
const WHITESPACE_RAW = 0.25;
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
function sepRawOf(sep: string): number {
return sep ? WHITESPACE_RAW : 0;
}
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
let start = 0;
let acc = 0;
let first = true;
const push = (end: number) => {
const text = word.slice(start, end);
out.push({
text,
raw: rawTokens(text),
sep: first ? sep : '',
sepRaw: first ? sepRawOf(sep) : 0,
strength: first ? strength : -1,
});
first = false;
start = end;
acc = 0;
};
for (let i = 0; i < word.length; i++) {
const c = word.charCodeAt(i);
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
acc += weight;
}
if (start < word.length) push(word.length);
}
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
const raw = rawTokens(sentence);
if (raw <= atomMax) {
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
return;
}
// Oversized sentence (or table row, or blob): fall back to groups of words.
const groupMax = Math.max(1, Math.min(atomMax, 24));
const words = sentence.split(/[ \t]+/);
// Keep leading indentation attached to the first word.
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
let group = '';
let groupRaw = 0;
let first = true;
const flush = () => {
if (!group) return;
out.push({
text: group,
raw: rawTokens(group),
sep: first ? sep : ' ',
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
strength: first ? strength : 0,
});
first = false;
group = '';
groupRaw = 0;
};
let pendingIndent = indent;
for (const word of words) {
if (!word) continue;
const piece = pendingIndent + word;
pendingIndent = '';
const wordRaw = rawTokens(piece);
if (wordRaw > atomMax) {
flush();
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
first = false;
continue;
}
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
group = group ? `${group} ${piece}` : piece;
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
}
flush();
}
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
function atomize(text: string, atomMax: number): Atom[] {
const atoms: Atom[] = [];
let prevLine = '';
for (const para of text.split(/\n{2,}/)) {
let firstInPara = true;
for (const rawLine of para.split('\n')) {
const line = rawLine.trimEnd();
if (!line.trim()) continue;
let sep = '';
let strength = 0;
if (atoms.length > 0) {
const afterHeading = HEADING_LINE.test(prevLine);
const isHeading = HEADING_LINE.test(line);
if (firstInPara) {
sep = '\n\n';
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
} else {
sep = '\n';
if (isHeading) strength = 4;
else if (afterHeading) strength = -0.5;
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
else strength = 0.5;
}
}
firstInPara = false;
prevLine = line;
if (rawTokens(line) <= Math.min(atomMax, 12)) {
// Short line: not worth looking for sentences.
pushSentence(line, atomMax, atoms, sep, strength);
continue;
}
let firstSentence = true;
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
if (!sentence) continue;
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
firstSentence = false;
}
}
}
return atoms;
}
function joinAtoms(atoms: Atom[], from: number, to: number): string {
let text = '';
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
return text;
}
/** Trailing whole words of `text` worth at most `budget` tokens. */
function tailWords(text: string, budget: number): string {
let end = text.length;
let cut = text.length;
let acc = 0;
while (end > 0) {
let wordEnd = end;
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
if (wordEnd === 0) break;
const space = text.lastIndexOf(' ', wordEnd - 1);
const word = text.slice(space + 1, wordEnd);
const cost = rawTokens(word) + WHITESPACE_RAW;
if (acc + cost > budget) break;
acc += cost;
cut = space + 1;
end = space < 0 ? 0 : space;
}
return text.slice(cut).trim();
}
/**
* Splits one oversized section into balanced pieces of at most `bodyBudget`
* tokens, cutting at the strongest boundary near the balanced size, and
* prefixing each later piece with up to `overlap` tokens of trailing context.
*/
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
const atomMax = Math.max(1, bodyBudget / 4);
const atoms = atomize(text, atomMax);
const n = atoms.length;
const pieces: string[] = [];
if (n === 0) return pieces;
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
const suffix = new Float64Array(n + 1);
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
let start = 0;
let prevStart = -1;
while (start < n) {
const remaining = suffix[start] - atoms[start].sepRaw;
let end: number;
if (remaining <= bodyBudget) {
end = n;
} else {
const goal = remaining / Math.ceil(remaining / bodyBudget);
const minFill = goal * 0.6;
let acc = 0;
let best = -1;
let bestStrength = -Infinity;
let bestDistance = Infinity;
let i = start;
for (; i < n; i++) {
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
if (i > start && acc + add > bodyBudget) break;
acc += add;
if (i + 1 < n && acc >= minFill) {
const strength = atoms[i + 1].strength;
const distance = Math.abs(acc - goal);
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
best = i;
bestStrength = strength;
bestDistance = distance;
}
}
}
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
}
let prefix = '';
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
let from = start;
let acc = 0;
while (from - 1 > prevStart) {
const candidate = atoms[from - 1];
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
if (acc + cost > overlap) break;
acc += cost;
from--;
if (candidate.strength >= 4) break; // do not reach back past a heading
}
if (from < start) prefix = joinAtoms(atoms, from, start);
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
}
const body = joinAtoms(atoms, start, end);
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
prevStart = start;
start = end;
}
return pieces;
}
function positiveInt(value: number | undefined, fallback: number, min: number): number {
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
return Math.max(min, Math.floor(value));
}
interface Pending {
text: string;
heading: string;
page: number | null;
}
/**
* Builds retrieval chunks from the sections of ONE document.
*
* - Consecutive small sections (short slides, short pages) are packed together
* up to `targetTokens`; the chunk keeps the first section's heading and the
* full page range.
* - A section that fits comfortably under the ceiling stays whole so chunks
* line up with pages/slides.
* - Larger sections are split into balanced pieces at paragraph, then
* sentence, then word boundaries, with `overlapTokens` of repeated context
* between consecutive pieces of the same section only.
*/
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
const target = positiveInt(opts.targetTokens, 450, 8);
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
/** Sections up to this size are kept in one piece. */
const wholeLimit = Math.floor(target + (max - target) / 2);
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
const chunks: DocChunk[] = [];
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
const trimmed = text.trim();
if (!trimmed) return;
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
};
let pending: Pending[] = [];
let pendingRaw = 0;
const flush = () => {
if (pending.length === 0) return;
let pageStart: number | null = null;
let pageEnd: number | null = null;
for (const item of pending) {
if (item.page === null) continue;
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
}
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
pending = [];
pendingRaw = 0;
};
let lastHeading = '';
for (const section of sections ?? []) {
if (!section || typeof section.text !== 'string') continue;
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
if (ownHeading) lastHeading = ownHeading;
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
if (!text) continue;
const heading = lastHeading;
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
const raw = rawTokens(text);
if (raw > wholeLimit) {
flush();
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
continue;
}
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
pending.push({ text, heading, page });
}
flush();
return chunks;
}

View file

@ -0,0 +1,420 @@
import { strToU8, zipSync } from 'fflate';
import { describe, expect, it } from 'vitest';
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
import { detectKind, extractDocument } from './extract.ts';
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
import { ExtractionError, type ExtractedSection } from './types.ts';
const enc = (s: string) => new TextEncoder().encode(s);
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
async function codeOf(promise: Promise<unknown>): Promise<string> {
try {
await promise;
} catch (e) {
expect(e).toBeInstanceOf(ExtractionError);
return (e as ExtractionError).code;
}
throw new Error('expected rejection');
}
describe('detectKind', () => {
const pdf = buildPdf([{ lines: ['hello'] }]);
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
const pptx = buildPptx([{ file: 1, title: 'T' }]);
it('detects by magic bytes', () => {
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
});
it('trusts magic bytes over a wrong extension', () => {
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
});
it('uses the extension for text and markdown', () => {
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
});
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
});
});
describe('extractDocument: PDF', () => {
it('extracts multi-page text with page numbers, count and title', async () => {
const bytes = buildPdf(
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
{ title: 'Cardiology Notes' },
);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.kind).toBe('pdf');
expect(r.pageCount).toBe(2);
expect(r.title).toBe('Cardiology Notes');
expect(r.needsOcr).toBe(false);
expect(r.warnings).toEqual([]);
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
expect(r.sections[0].text).toContain('Heart failure overview');
expect(r.sections[1].text).toContain('Digoxin');
});
it('warns about a page without text but keeps page numbers', async () => {
const long = 'Readable text on this page, long enough to avoid OCR. ';
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.pageCount).toBe(3);
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
});
it('flags an all-empty scanned PDF as needing OCR', async () => {
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
expect(r.needsOcr).toBe(true);
expect(r.sections).toEqual([]);
expect(r.pageCount).toBe(2);
expect(r.warnings.join(' ')).toMatch(/scanned/);
});
it('reports progress per page', async () => {
const seen: Array<[number, number]> = [];
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
expect(seen).toEqual([
[1, 2],
[2, 2],
]);
});
});
describe('extractDocument: DOCX', () => {
it('turns headings into sections and keeps lists and tables readable', async () => {
const bytes = buildDocx(
[
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
{ type: 'p', text: 'Intro paragraph about drugs.' },
{ type: 'bullet', text: 'Beta blockers' },
{ type: 'bullet', text: 'ACE inhibitors' },
{ type: 'heading', level: 2, text: 'Dosing' },
{ type: 'number', text: 'Check heart rate' },
{ type: 'number', text: 'Give the dose' },
{
type: 'table',
rows: [
['Drug', 'Dose'],
['Digoxin', '0.25 mg'],
],
},
],
{ title: 'Drug Guide' },
);
const r = await extractDocument({ bytes, filename: 'a.docx' });
expect(r.kind).toBe('docx');
expect(r.title).toBe('Drug Guide');
expect(r.needsOcr).toBe(false);
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
const first = r.sections[0].text;
expect(first).toContain('Intro paragraph');
expect(first).toContain('Beta blockers');
expect(first).toContain('ACE inhibitors');
const second = r.sections[1].text;
expect(second).toContain('Check heart rate');
expect(second).toContain('Digoxin');
expect(second).toContain('0.25 mg');
});
it('rejects a docx with no text as empty', async () => {
const bytes = buildDocx([{ type: 'p', text: '' }]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
});
});
describe('extractDocument: PPTX', () => {
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
const bytes = buildPptx(
[
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
{ file: 2 },
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
],
{ title: 'Deck Title' },
);
const r = await extractDocument({ bytes, filename: 'a.pptx' });
expect(r.kind).toBe('pptx');
expect(r.title).toBe('Deck Title');
expect(r.pageCount).toBe(4);
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
expect(r.sections[0].text).toContain('Welcome');
expect(r.sections[0].text).toContain('Sub point');
expect(r.sections[0].text).toContain('Say hello slowly');
expect(r.sections[1].text).toContain('Free text box content here');
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
});
it('reads tables', async () => {
const table = [
['A', 'B'],
['x', 'y'],
];
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
expect(allText(r)).toContain('x');
expect(allText(r)).toContain('B');
});
it('warns when most slides have no text', async () => {
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
});
});
describe('extractDocument: text, markdown, image', () => {
it('splits markdown by heading and ignores headings inside code fences', async () => {
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
expect(r.kind).toBe('markdown');
expect(r.title).toBe('Title');
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
expect(r.sections[1].text).toContain('# not a heading');
expect(r.needsOcr).toBe(false);
});
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
expect(r.kind).toBe('text');
expect(allText(r)).toContain('line two');
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
expect(allText(r2)).toBe('café');
expect(r2.warnings.length).toBe(1);
});
it('groups long plain text into several bounded sections', async () => {
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
expect(r.sections.length).toBeGreaterThan(1);
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
});
it('returns images with needsOcr and no sections', async () => {
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
});
});
describe('extractDocument: errors', () => {
it('rejects an empty file', async () => {
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
});
it('rejects whitespace-only text as empty', async () => {
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
});
it('rejects unsupported types and legacy .doc', async () => {
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
});
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
const marker = Buffer.from('EncryptedPackage', 'utf16le');
const bytes = Uint8Array.from([...OLE, ...marker]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
});
it('rejects corrupt PDFs', async () => {
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
});
it('rejects an encrypted PDF', async () => {
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
});
it('rejects a corrupt zip posing as docx/pptx', async () => {
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
const broken = good.slice(0, Math.floor(good.length / 2));
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a pptx zip that has no readable slides', async () => {
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a zip bomb whose headers lie about entry size', async () => {
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
expect(['too_large', 'corrupt']).toContain(code);
}, 30000);
it('honours an already-aborted signal', async () => {
const ac = new AbortController();
ac.abort();
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
});
});
describe('chunkSections', () => {
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
page: i + 1,
}));
const chunks = chunkSections(sections, opts);
expect(chunks.length).toBeGreaterThan(12);
chunks.forEach((c, i) => {
expect(c.seq).toBe(i);
expect(c.tokens).toBe(estimateTokens(c.text));
expect(c.tokens).toBeLessThanOrEqual(150);
});
});
it('packs small sections together with correct page range and first heading', () => {
const sections: ExtractedSection[] = [
{ text: 'Alpha text.', page: 1, heading: 'First' },
{ text: 'Beta text.', page: 2 },
{ text: 'Gamma text.', page: 3, heading: 'Third' },
];
const chunks = chunkSections(sections, opts);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
expect(chunks[0].text).toContain('Gamma');
});
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(c.pageStart).not.toBeNull();
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
}
expect(chunks[0].pageStart).toBe(1);
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
});
it('uses null pages and empty heading when sections have none', () => {
const [c] = chunkSections([{ text: 'No pages here.' }]);
expect(c.pageStart).toBeNull();
expect(c.pageEnd).toBeNull();
expect(c.heading).toBe('');
});
it('splits oversized sections without cutting words', () => {
const text = words(800, 'term');
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
expect(chunks.length).toBeGreaterThan(3);
const vocabulary = new Set(text.split(' '));
for (const c of chunks) {
expect(c.pageStart).toBe(5);
expect(c.pageEnd).toBe(5);
expect(c.heading).toBe('Big');
expect(c.tokens).toBeLessThanOrEqual(150);
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
}
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
expect(seen.size).toBe(800);
});
it('survives a giant unbroken string within the limit', () => {
const blob = 'x'.repeat(200_000);
const t0 = Date.now();
const chunks = chunkSections([{ text: blob }], opts);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
});
it('does not split surrogate pairs in unbroken emoji runs', () => {
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(wellFormed(c.text)).toBe(true);
expect(c.tokens).toBeLessThanOrEqual(150);
}
});
it('is deterministic', () => {
const sections = [
{ text: words(900), page: 1, heading: 'H' },
{ text: 'Short one.', page: 2 },
];
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
});
it('drops empty and whitespace-only sections', () => {
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
const chunks = chunkSections([
{ text: ' ', page: 1 },
{ text: 'Real content.', page: 2 },
{ text: '\n', page: 3 },
]);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
});
it('chunks about 2 MB of text quickly', () => {
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
const t0 = Date.now();
const chunks = chunkSections([{ text }]);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(100);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
});
});
describe('cleanText', () => {
it('joins words hyphenated across a line break', () => {
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
});
it('keeps real compounds across a line break', () => {
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
});
it('collapses 3+ newlines to a paragraph break', () => {
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
expect(cleanText('a\n\nb')).toBe('a\n\nb');
});
it('strips control characters but keeps tabs and newlines', () => {
expect(cleanText('he\u0000l\u0007lo​ ­world\ttab\nnext')).toBe('hello world\ttab\nnext');
});
it('normalises CRLF and trims', () => {
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
expect(cleanText('')).toBe('');
});
it('is idempotent', () => {
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny  z');
expect(cleanText(once)).toBe(once);
});
});

105
server/documents/docx.ts Normal file
View file

@ -0,0 +1,105 @@
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
import { zipSync } from 'fflate';
import mammoth from 'mammoth';
import { cleanText } from './chunk.ts';
import { htmlToBlocks } from './markup.ts';
import { readCoreTitle, type OfficeContext } from './pptx.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader } from './zip.ts';
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
const MAX_WARNINGS = 3;
/**
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
* into a fresh stored zip. Every byte went through the size-limited reader, so
* mammoth's own unzip can never be handed a bomb, and it never spends time
* base64-encoding images we would throw away.
*/
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
const files: Record<string, Uint8Array> = {};
let images = 0;
for (const entry of zip.entries) {
const name = entry.name;
if (name.endsWith('/')) continue;
if (/^word\/media\//i.test(name)) {
images++;
continue;
}
if (!/\.(xml|rels)$/i.test(name)) continue;
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
const data = zip.read(name);
if (data) files[name] = data;
}
return { data: zipSync(files, { level: 0 }), images };
}
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
const seen = new Set<string>();
for (const item of messages) {
const message = String(item?.message ?? '').trim();
// Unknown style names never lose text; they are noise for a reader.
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
}
const unique = [...seen];
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
return out;
}
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
}
const { data, images } = sanitisedPackage(zip);
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const converted = await mammoth.convertToHtml(
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
{
styleMap: STYLE_MAP,
ignoreEmptyParagraphs: true,
// Images are not read locally; never decode them.
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
},
);
const sections: ExtractedSection[] = [];
let heading: string | undefined;
let parts: string[] = [];
const flush = () => {
const text = cleanText(parts.join('\n\n'));
parts = [];
if (!text) return;
const section: ExtractedSection = { text };
if (heading) section.heading = heading;
sections.push(section);
};
for (const block of htmlToBlocks(converted.value)) {
if (block.level >= 1 && block.level <= 3) {
flush();
heading = block.text.slice(0, 200);
}
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
}
flush();
ctx.onProgress?.(1, 1);
const warnings = friendlyWarnings(converted.messages ?? []);
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
let needsOcr = false;
if (totalChars < 25) {
if (images === 0) {
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
}
needsOcr = true;
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
}
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

362
server/documents/extract.ts Normal file
View file

@ -0,0 +1,362 @@
/**
* Document text extraction: kind detection plus one extractor per format.
* Every failure surfaces as an ExtractionError with a user-facing message.
*/
import { cleanText } from './chunk.ts';
import { extractDocx } from './docx.ts';
import { extractPdf } from './pdf.ts';
import { extractPptx } from './pptx.ts';
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
import { listZip, ZipError, type ZipLimits } from './zip.ts';
/** Largest upload accepted (100 MB). */
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
export interface ExtractInput {
bytes: Uint8Array;
filename: string;
mime?: string;
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
onProgress?: (done: number, total: number) => void;
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
signal?: AbortSignal;
}
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
const SNIFF_BYTES = 256 * 1024;
function extensionOf(filename: string): string {
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
}
function baseMime(mime: string | undefined): string {
return (mime ?? '').split(';')[0].trim().toLowerCase();
}
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
if (bytes.length < offset + signature.length) return false;
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
return true;
}
function isImage(bytes: Uint8Array): boolean {
return (
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
);
}
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
function hasPdfHeader(bytes: Uint8Array): boolean {
const limit = Math.min(bytes.length - 5, 1024);
for (let i = 0; i <= limit; i++) {
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
}
return false;
}
function isZip(bytes: Uint8Array): boolean {
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
}
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
function isOle(bytes: Uint8Array): boolean {
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
}
function isUtf16Bom(bytes: Uint8Array): boolean {
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
}
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
try {
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
return true;
} catch {
return false;
}
}
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
function looksLikeSingleByteText(sample: Uint8Array): boolean {
let high = 0;
for (let i = 0; i < sample.length; i++) {
const b = sample[i];
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
if (b === 0x7f) return false;
if (b >= 0x80) high++;
}
return high <= sample.length * 0.3;
}
function isProbablyText(bytes: Uint8Array): boolean {
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
if (sample.includes(0)) return false;
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
}
/**
* Works out what a file is. Magic bytes win over the name; the extension and
* MIME type are only a fallback. Returns null for anything unsupported
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
*/
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
const ext = extensionOf(filename);
const type = baseMime(mime);
if (!bytes || bytes.length === 0) return null;
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
if (isImage(bytes)) return 'image';
if (isZip(bytes)) {
let names: string[] | null = null;
try {
names = listZip(bytes).map((entry) => entry.name);
} catch {
names = null;
}
if (names) {
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
if (names.includes('ppt/presentation.xml')) return 'pptx';
return null; // xlsx, odt, epub, jar, plain zip...
}
// Unreadable archive: trust the name so extraction can report it as damaged.
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
return null;
}
if (isOle(bytes)) {
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
if (ext === 'docx') return 'docx';
if (ext === 'pptx') return 'pptx';
return null;
}
// No recognisable signature: fall back to the name / declared type.
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
return null;
}
// ---------------------------------------------------------------------------
// Text and Markdown
// ---------------------------------------------------------------------------
function decodeText(bytes: Uint8Array, warnings: string[]): string {
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
return new TextDecoder('utf-8').decode(bytes.subarray(3));
}
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
try {
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch {
if (bytes.includes(0)) {
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
}
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
return new TextDecoder('windows-1252').decode(bytes);
}
}
/** Target size of a plain-text section, in characters. */
const TEXT_SECTION_CHARS = 6000;
function splitPlainText(text: string): ExtractedSection[] {
const sections: ExtractedSection[] = [];
let current: string[] = [];
let size = 0;
const flush = (separator: string) => {
if (current.length === 0) return;
sections.push({ text: current.join(separator) });
current = [];
size = 0;
};
for (const block of text.split(/\n{2,}/)) {
if (!block.trim()) continue;
if (block.length > TEXT_SECTION_CHARS) {
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
flush('\n\n');
for (const line of block.split('\n')) {
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
current.push(line);
size += line.length + 1;
}
flush('\n');
continue;
}
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
current.push(block);
size += block.length + 2;
}
flush('\n\n');
return sections;
}
function markdownHeadingText(raw: string): string {
return raw
.replace(/[ \t]+#+[ \t]*$/, '')
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\*\*|__|[*`]/g, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
}
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
const sections: ExtractedSection[] = [];
let title: string | undefined;
let heading: string | undefined;
let lines: string[] = [];
let fence: string | null = null;
const flush = () => {
const body = lines.join('\n').trim();
lines = [];
if (!body) return;
const section: ExtractedSection = { text: body };
if (heading) section.heading = heading;
sections.push(section);
};
for (const line of text.split('\n')) {
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
if (fenceMatch) {
const marker = fenceMatch[1][0];
if (fence === null) fence = marker;
else if (fence === marker) fence = null;
} else if (fence === null && line.length < 400) {
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
if (match) {
const headingText = markdownHeadingText(match[2]);
if (headingText) {
flush();
heading = headingText;
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
}
}
}
lines.push(line);
}
flush();
return { sections, title };
}
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
const warnings: string[] = [];
const text = cleanText(decodeText(bytes, warnings));
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
if (kind === 'markdown') {
const { sections, title } = splitMarkdown(text);
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
if (title) result.title = title;
return result;
}
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
const ext = extensionOf(filename);
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
}
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
}
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
}
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
if (error.code === 'too_large') {
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
}
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
}
/**
* Extracts the text of a document as ordered sections. Images return no
* sections and `needsOcr: true`; the caller runs vision transcription.
*/
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
const bytes = input?.bytes;
const filename = typeof input?.filename === 'string' ? input.filename : '';
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
}
if (bytes.length > MAX_DOCUMENT_BYTES) {
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
}
if (input.signal?.aborted) {
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
}
let kind: DocKind | null;
try {
kind = detectKind(filename, input.mime, bytes);
} catch (error) {
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
}
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
try {
switch (kind) {
case 'pdf':
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
case 'docx':
case 'pptx': {
if (isOle(bytes)) {
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
}
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
}
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
}
case 'text':
case 'markdown': {
const result = extractText(bytes, kind);
input.onProgress?.(1, 1);
return result;
}
case 'image':
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
}
} catch (error) {
if (error instanceof ExtractionError) throw error;
if (error instanceof ZipError) throw fromZipError(error, kind);
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
if (kind === 'docx' || kind === 'pptx') {
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
}
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
}
}

272
server/documents/markup.ts Normal file
View file

@ -0,0 +1,272 @@
/** Small markup helpers shared by the DOCX and PPTX extractors. */
const NAMED_ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
};
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
export function decodeEntities(text: string): string {
if (!text.includes('&')) return text;
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
return String.fromCodePoint(code);
}
return NAMED_ENTITIES[body] ?? match;
});
}
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
export function renderTable(rows: string[][]): string {
const cleaned = rows
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
.filter((row) => row.some((cell) => cell.length > 0));
if (cleaned.length === 0) return '';
const width = Math.max(...cleaned.map((row) => row.length));
const line = (row: string[]) => {
const cells = row.slice();
while (cells.length < width) cells.push('');
return `| ${cells.join(' | ')} |`;
};
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
return out.join('\n');
}
// ---------------------------------------------------------------------------
// HTML (as produced by mammoth) -> Markdown blocks
// ---------------------------------------------------------------------------
export interface MarkdownBlock {
/** 1-6 for headings, 0 for everything else. */
level: number;
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
text: string;
}
interface HtmlElement {
tag: string;
children: HtmlNode[];
}
type HtmlNode = HtmlElement | string;
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
const HEADING_TAG = /^h[1-6]$/;
function parseHtml(html: string): HtmlElement {
const root: HtmlElement = { tag: '#root', children: [] };
const stack: HtmlElement[] = [root];
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
let match: RegExpExecArray | null;
while ((match = tokens.exec(html)) !== null) {
const top = stack[stack.length - 1];
if (match[4] !== undefined) {
top.children.push(decodeEntities(match[4]));
} else if (match[2] === undefined) {
top.children.push('<');
} else {
const tag = match[2].toLowerCase();
if (match[1]) {
// Close the nearest matching open element; ignore stray closers.
for (let i = stack.length - 1; i > 0; i--) {
if (stack[i].tag === tag) {
stack.length = i;
break;
}
}
} else {
const element: HtmlElement = { tag, children: [] };
top.children.push(element);
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
}
}
}
return root;
}
function collapse(text: string): string {
return text.replace(/[ \t\r\n]+/g, ' ');
}
function wrap(marker: string, inner: string): string {
const core = inner.trim();
if (!core) return inner;
const lead = inner.slice(0, inner.length - inner.trimStart().length);
const trail = inner.slice(inner.trimEnd().length);
return `${lead}${marker}${core}${marker}${trail}`;
}
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
function inlineText(nodes: HtmlNode[], plain: boolean): string {
let out = '';
for (const node of nodes) {
if (typeof node === 'string') {
out += collapse(node);
continue;
}
switch (node.tag) {
case 'br':
out += '\n';
break;
case 'img':
case 'hr':
break;
case 'strong':
case 'b':
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
break;
case 'em':
case 'i':
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
break;
case 'a': {
const text = inlineText(node.children, plain);
// mammoth's footnote back-references
if (text.trim() !== '↑') out += text;
break;
}
case 'p':
case 'li':
case 'tr':
case 'td':
case 'th':
case 'ul':
case 'ol':
case 'table': {
const inner = inlineText(node.children, plain).trim();
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
break;
}
default:
out += inlineText(node.children, plain);
}
}
return out;
}
function tidyInline(text: string): string {
return text
.split('\n')
.map((line) => line.replace(/ {2,}/g, ' ').trim())
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
.join('\n')
.trim();
}
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
const ordered = list.tag === 'ol';
let counter = 0;
for (const item of list.children) {
if (typeof item === 'string' || item.tag !== 'li') continue;
counter++;
const marker = ordered ? `${counter}. ` : '- ';
const inlineNodes: HtmlNode[] = [];
const nested: HtmlElement[] = [];
const paragraphs: string[] = [];
const flushInline = () => {
const text = tidyInline(inlineText(inlineNodes, false));
if (text) paragraphs.push(text);
inlineNodes.length = 0;
};
for (const child of item.children) {
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
flushInline();
nested.push(child);
} else if (typeof child !== 'string' && child.tag === 'p') {
flushInline();
inlineNodes.push(...child.children);
flushInline();
} else if (typeof child !== 'string' && child.tag === 'table') {
flushInline();
const table = renderHtmlTable(child);
if (table) paragraphs.push(table);
} else {
inlineNodes.push(child);
}
}
flushInline();
const pad = indent + ' '.repeat(marker.length);
const body = paragraphs.join('\n').split('\n');
if (body.length > 0 && body[0] !== '') {
lines.push(indent + marker + body[0]);
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
} else if (nested.length === 0) {
continue;
}
for (const child of nested) renderList(child, pad, lines);
}
}
function renderHtmlTable(table: HtmlElement): string {
const rows: string[][] = [];
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') continue;
if (node.tag === 'tr') {
const cells: string[] = [];
for (const cell of node.children) {
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
cells.push(tidyInline(inlineText(cell.children, false)));
}
rows.push(cells);
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
visit(node.children);
}
}
};
visit(table.children);
return renderTable(rows);
}
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
export function htmlToBlocks(html: string): MarkdownBlock[] {
const root = parseHtml(html);
const blocks: MarkdownBlock[] = [];
const loose: HtmlNode[] = [];
const flushLoose = () => {
if (loose.length === 0) return;
const text = tidyInline(inlineText(loose, false));
if (text) blocks.push({ level: 0, text });
loose.length = 0;
};
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') {
loose.push(node);
continue;
}
if (HEADING_TAG.test(node.tag)) {
flushLoose();
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
if (text) blocks.push({ level: Number(node.tag[1]), text });
} else if (node.tag === 'p') {
flushLoose();
const text = tidyInline(inlineText(node.children, false));
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'ul' || node.tag === 'ol') {
flushLoose();
const lines: string[] = [];
renderList(node, '', lines);
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
} else if (node.tag === 'table') {
flushLoose();
const text = renderHtmlTable(node);
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
flushLoose();
visit(node.children);
} else {
loose.push(node);
}
}
};
visit(root.children);
flushLoose();
return blocks;
}

249
server/documents/pdf.ts Normal file
View file

@ -0,0 +1,249 @@
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
import { getDocumentProxy } from 'unpdf';
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
export interface PdfContext {
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
/** Below this average of characters per page the PDF is treated as a scan. */
const OCR_CHARS_PER_PAGE = 25;
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
const CLEANUP_EVERY = 40;
interface TextItemLike {
str?: unknown;
hasEOL?: unknown;
transform?: unknown;
height?: unknown;
}
interface Line {
text: string;
y: number | null;
height: number;
}
/**
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
* order (which keeps multi-column pages readable) with `hasEOL` marking line
* ends; the vertical gaps between lines are then used to restore paragraph
* breaks, which pdf.js does not report.
*/
export function textItemsToText(items: readonly TextItemLike[]): string {
const lines: Line[] = [];
let text = '';
let y: number | null = null;
let height = 0;
const endLine = () => {
lines.push({ text, y, height });
text = '';
y = null;
height = 0;
};
for (const item of items) {
if (!item || typeof item.str !== 'string') continue;
const str = item.str;
if (str.length > 0) {
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
const t = item.transform as number[];
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
if (upright && Number.isFinite(t[5])) {
y = t[5];
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
height = Number.isFinite(h) ? h : 0;
}
}
text += str;
}
if (item.hasEOL === true) endLine();
}
if (text.length > 0) endLine();
// Typical line advance on this page (median of the downward steps).
const advances: number[] = [];
for (let i = 1; i < lines.length; i++) {
const a = lines[i - 1];
const b = lines[i];
if (a.y === null || b.y === null) continue;
const dy = a.y - b.y;
const ref = Math.max(a.height, b.height, 1);
if (dy > 0 && dy < ref * 4) advances.push(dy);
}
advances.sort((p, q) => p - q);
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
let out = '';
let prev: Line | null = null;
for (const line of lines) {
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
if (!content) {
// An empty line is an explicit gap.
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
continue;
}
if (out && !out.endsWith('\n')) {
let gap = false;
if (prev && prev.y !== null && line.y !== null) {
const dy = prev.y - line.y;
const ref = Math.max(prev.height, line.height, 1);
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
else gap = dy > ref * 1.9;
}
out += gap ? '\n\n' : '\n';
}
out += content;
prev = line;
}
return out;
}
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
export function tidyTitle(raw: unknown): string | undefined {
if (typeof raw !== 'string') return undefined;
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
return title;
}
function errorName(error: unknown): string {
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
}
function toExtractionError(error: unknown): ExtractionError {
if (error instanceof ExtractionError) return error;
const name = errorName(error);
const message = error instanceof Error ? error.message : String(error);
if (name === 'PasswordException' || /password/i.test(message)) {
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
}
if (
name === 'InvalidPDFException' ||
name === 'FormatError' ||
name === 'XRefParseException' ||
name === 'XRefEntryException' ||
name === 'MissingDataException' ||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
) {
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
}
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
}
function cancelled(signal: AbortSignal): ExtractionError {
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
}
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
const { onProgress, signal } = ctx;
if (signal?.aborted) throw cancelled(signal);
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
try {
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
pdf = await getDocumentProxy(bytes.slice(), options);
} catch (error) {
throw toExtractionError(error);
}
try {
const total = pdf.numPages;
if (!Number.isFinite(total) || total < 1) {
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
}
let title: string | undefined;
try {
const meta = await pdf.getMetadata();
const info = (meta?.info ?? {}) as Record<string, unknown>;
title = tidyTitle(info.Title);
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
} catch {
// Metadata is optional.
}
const pageTexts: string[] = new Array(total).fill('');
let failedPages = 0;
let lastYield = Date.now();
for (let n = 1; n <= total; n++) {
if (signal?.aborted) throw cancelled(signal);
try {
const page = await pdf.getPage(n);
try {
const content = await page.getTextContent();
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
} finally {
page.cleanup();
}
} catch (error) {
if (errorName(error) === 'PasswordException') throw error;
failedPages++;
}
if (n % CLEANUP_EVERY === 0) {
try {
await pdf.cleanup();
} catch {
// Best effort.
}
}
onProgress?.(n, total);
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
if (failedPages === total) {
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
}
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
const sections: ExtractedSection[] = [];
let empty = 0;
let chars = 0;
for (let i = 0; i < total; i++) {
const text = cleaned[i];
if (!text) {
empty++;
continue;
}
chars += text.length;
sections.push({ text, page: i + 1 });
}
const warnings: string[] = [];
if (empty > 0) {
warnings.push(
empty === total
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
: `${empty} of ${total} pages had no extractable text.`,
);
}
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
const result: ExtractResult = {
kind: 'pdf',
sections,
pageCount: total,
warnings,
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
};
if (title) result.title = title;
return result;
} catch (error) {
throw toExtractionError(error);
} finally {
try {
await pdf.loadingTask.destroy();
} catch {
// Nothing useful to do.
}
}
}

424
server/documents/pptx.ts Normal file
View file

@ -0,0 +1,424 @@
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
import { XMLParser } from 'fast-xml-parser';
import { cleanText } from './chunk.ts';
import { decodeEntities, renderTable } from './markup.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader, type ZipLimits } from './zip.ts';
export interface OfficeContext {
limits: ZipLimits;
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
type XmlNode = Record<string, unknown>;
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '',
preserveOrder: true,
trimValues: false,
parseTagValue: false,
parseAttributeValue: false,
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
processEntities: false,
ignoreDeclaration: true,
ignorePiTags: true,
});
function parseXml(xml: string): XmlNode[] {
const parsed = parser.parse(xml) as unknown;
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
}
function tagOf(node: XmlNode): string {
for (const key in node) if (key !== ':@') return key;
return '';
}
/** Element name without its namespace prefix. */
function localName(node: XmlNode): string {
const tag = tagOf(node);
const colon = tag.indexOf(':');
return colon < 0 ? tag : tag.slice(colon + 1);
}
function childrenOf(node: XmlNode): XmlNode[] {
const value = node[tagOf(node)];
return Array.isArray(value) ? (value as XmlNode[]) : [];
}
function attr(node: XmlNode, name: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
const direct = attrs[name];
if (typeof direct === 'string') return direct;
return undefined;
}
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
function attrLocal(node: XmlNode, local: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
for (const key in attrs) {
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
}
return undefined;
}
function child(node: XmlNode, local: string): XmlNode | undefined {
for (const c of childrenOf(node)) if (localName(c) === local) return c;
return undefined;
}
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
if (depth > 40) return undefined;
for (const node of nodes) {
if (localName(node) === local) return node;
const found = findDeep(childrenOf(node), local, depth + 1);
if (found) return found;
}
return undefined;
}
/** Concatenated text of every a:t / m:t descendant. */
function deepText(node: XmlNode, depth = 0): string {
if (depth > 40) return '';
const name = localName(node);
if (name === '#text') return '';
if (name === 't') {
let out = '';
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
return decodeEntities(out);
}
let out = '';
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
return out;
}
interface Paragraph {
text: string;
level: number;
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
bullet: boolean | undefined;
}
function readParagraph(p: XmlNode): Paragraph {
let text = '';
let level = 0;
let bullet: boolean | undefined;
for (const c of childrenOf(p)) {
switch (localName(c)) {
case 'pPr': {
const lvl = Number(attr(c, 'lvl'));
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
for (const b of childrenOf(c)) {
const name = localName(b);
if (name === 'buNone') bullet = false;
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
}
break;
}
case 'br':
text += '\n';
break;
case 'fld': {
const type = attr(c, 'type') ?? '';
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
break;
}
case 'endParaRPr':
case '#text':
break;
default:
text += deepText(c);
}
}
text = text
.replace(/[\u000B\u2028]/g, '\n')
.split('\n')
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
.filter((line) => line.length > 0)
.join('\n');
return { text, level, bullet };
}
function readParagraphs(txBody: XmlNode): Paragraph[] {
const out: Paragraph[] = [];
for (const c of childrenOf(txBody)) {
if (localName(c) !== 'p') continue;
const paragraph = readParagraph(c);
if (paragraph.text) out.push(paragraph);
}
return out;
}
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
const lines: string[] = [];
for (const p of paragraphs) {
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
if (!bulleted) {
lines.push(p.text);
continue;
}
const indent = ' '.repeat(p.level);
const parts = p.text.split('\n');
lines.push(`${indent}- ${parts[0]}`);
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
}
return lines.join('\n');
}
function readTable(tbl: XmlNode): string {
const rows: string[][] = [];
for (const tr of childrenOf(tbl)) {
if (localName(tr) !== 'tr') continue;
const cells: string[] = [];
for (const tc of childrenOf(tr)) {
if (localName(tc) !== 'tc') continue;
const txBody = child(tc, 'txBody');
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
}
rows.push(cells);
}
return renderTable(rows);
}
interface SlideContent {
title: string;
blocks: string[];
}
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
function placeholderType(shape: XmlNode): string | null {
const nv = child(shape, 'nvSpPr');
const nvPr = nv ? child(nv, 'nvPr') : undefined;
const ph = nvPr ? child(nvPr, 'ph') : undefined;
if (!ph) return null;
return attr(ph, 'type') ?? 'body';
}
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
if (depth > 30) return;
for (const node of nodes) {
switch (localName(node)) {
case 'sp': {
const type = placeholderType(node);
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
const txBody = child(node, 'txBody');
if (!txBody) break;
const paragraphs = readParagraphs(txBody);
if (paragraphs.length === 0) break;
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
} else {
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
const text = renderParagraphs(paragraphs, bullets);
if (text) into.blocks.push(text);
}
break;
}
case 'grpSp':
walkShapes(childrenOf(node), into, depth + 1);
break;
case 'graphicFrame': {
const tbl = findDeep(childrenOf(node), 'tbl');
if (tbl) {
const table = readTable(tbl);
if (table) into.blocks.push(table);
}
break;
}
case 'AlternateContent': {
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
break;
}
default:
break;
}
}
}
function readSlide(xml: string): SlideContent {
const content: SlideContent = { title: '', blocks: [] };
const spTree = findDeep(parseXml(xml), 'spTree');
if (spTree) walkShapes(childrenOf(spTree), content, 0);
return content;
}
function readNotes(xml: string): string {
const spTree = findDeep(parseXml(xml), 'spTree');
if (!spTree) return '';
const parts: string[] = [];
for (const shape of childrenOf(spTree)) {
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
const txBody = child(shape, 'txBody');
if (!txBody) continue;
const text = renderParagraphs(readParagraphs(txBody), false);
if (text) parts.push(text);
}
return parts.join('\n');
}
interface Relationship {
id: string;
type: string;
target: string;
external: boolean;
}
function readRelationships(xml: string | null): Relationship[] {
if (!xml) return [];
const out: Relationship[] = [];
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
let match: RegExpExecArray | null;
while ((match = tags.exec(xml)) !== null) {
const attrs: Record<string, string> = {};
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
let pair: RegExpExecArray | null;
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
if (attrs.Id && attrs.Target) {
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
}
}
return out;
}
/** Resolves a relationship target against the directory of the part that owns it. */
function resolveTarget(baseDir: string, target: string): string {
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
for (const segment of target.split('/')) {
if (!segment || segment === '.') continue;
if (segment === '..') parts.pop();
else parts.push(segment);
}
return parts.join('/');
}
function relsPathFor(part: string): string {
const slash = part.lastIndexOf('/');
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
}
function slideOrder(zip: ZipReader): string[] {
const ordered: string[] = [];
const presentation = zip.readText('ppt/presentation.xml');
if (presentation) {
const targets = new Map<string, string>();
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
}
const list = findDeep(parseXml(presentation), 'sldIdLst');
if (list) {
for (const item of childrenOf(list)) {
if (localName(item) !== 'sldId') continue;
const id = attrLocal(item, 'id');
const path = id ? targets.get(id) : undefined;
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
}
}
}
if (ordered.length > 0) return ordered;
// Damaged or unusual package: fall back to numeric file order.
return zip
.names()
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
.map((item) => item.name);
}
export function readCoreTitle(zip: ZipReader): string | undefined {
let xml: string | null = null;
try {
xml = zip.readText('docProps/core.xml');
} catch {
return undefined;
}
if (!xml) return undefined;
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
return title || undefined;
}
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
const slides = slideOrder(zip);
if (slides.length === 0) {
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
}
const sections: ExtractedSection[] = [];
const warnings: string[] = [];
let withoutText = 0;
let unreadable = 0;
let totalChars = 0;
let lastYield = Date.now();
for (let i = 0; i < slides.length; i++) {
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const path = slides[i];
let text = '';
let title = '';
try {
const xml = zip.readText(path);
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
title = slide.title;
const parts: string[] = [];
if (slide.title) parts.push(`## ${slide.title}`);
parts.push(...slide.blocks);
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
);
if (notesRel) {
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
const notes = notesXml ? readNotes(notesXml) : '';
if (notes) parts.push(`Notes:\n${notes}`);
}
text = cleanText(parts.join('\n\n'));
} catch (error) {
if (error instanceof ExtractionError) throw error;
if ((error as { code?: string })?.code === 'too_large') throw error;
unreadable++;
}
if (text) {
totalChars += text.length;
const section: ExtractedSection = { text, page: i + 1 };
if (title) section.heading = title;
sections.push(section);
} else {
withoutText++;
}
ctx.onProgress?.(i + 1, slides.length);
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
const total = slides.length;
if (unreadable === total) {
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
}
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
if (withoutText === total) {
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
} else if (withoutText * 2 > total) {
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
}
const result: ExtractResult = {
kind: 'pptx',
sections,
pageCount: total,
warnings,
needsOcr: totalChars < total * 25,
};
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

View file

@ -0,0 +1,270 @@
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
import { strToU8, zipSync } from 'fflate';
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
function pdfString(text: string): string {
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
}
export interface PdfPage {
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
lines?: string[];
/** Raw content stream, used instead of `lines`. */
raw?: string;
}
export interface PdfOptions {
title?: string;
/** Adds a standard security handler whose user password is not empty. */
encrypted?: boolean;
}
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
const objects: string[] = [];
const add = (body: string): number => {
objects.push(body);
return objects.length;
};
const catalogId = add('');
const pagesId = add('');
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
const pageIds: number[] = [];
for (const page of pages) {
let stream = page.raw ?? '';
if (page.raw === undefined && page.lines && page.lines.length > 0) {
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
for (const line of page.lines) {
if (line === '') ops.push('0 -14 Td');
else ops.push(`${pdfString(line)} Tj`, 'T*');
}
ops.push('ET');
stream = ops.join('\n');
}
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
pageIds.push(
add(
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
),
);
}
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
let infoId = 0;
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
let encryptId = 0;
if (options.encrypted) {
const o = 'a1'.repeat(32);
const u = 'b2'.repeat(32);
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
}
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
const offsets: number[] = [];
objects.forEach((body, index) => {
offsets.push(out.length);
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
});
const xrefOffset = out.length;
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
if (infoId) out += ` /Info ${infoId} 0 R`;
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
return latin1(out);
}
const xmlEscape = (text: string): string => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
function coreProps(title: string | undefined): string {
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
}
// ---------------------------------------------------------------------------
// DOCX
// ---------------------------------------------------------------------------
export type DocxBlock =
| { type: 'heading'; level: number; text: string }
| { type: 'p'; text: string; bold?: string }
| { type: 'bullet'; text: string; level?: number }
| { type: 'number'; text: string; level?: number }
| { type: 'table'; rows: string[][] }
| { type: 'image' };
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
function docxRun(text: string, bold = false): string {
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
}
function docxBlock(block: DocxBlock): string {
switch (block.type) {
case 'heading':
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
case 'p':
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
case 'bullet':
case 'number':
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
case 'table':
return `<w:tbl>${block.rows
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
.join('')}</w:tbl>`;
case 'image':
return '<w:p/>';
}
}
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
const levels = (format: string) =>
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
),
'word/_rels/document.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
),
'word/numbering.xml': strToU8(
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
),
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
'docProps/core.xml': strToU8(coreProps(options.title)),
...options.extraFiles,
};
return zipSync(files);
}
// ---------------------------------------------------------------------------
// PPTX
// ---------------------------------------------------------------------------
export interface PptxSlide {
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
file: number;
title?: string;
/** Body placeholder paragraphs. */
bullets?: Array<string | { text: string; level: number }>;
/** A free text box (not a placeholder). */
textBox?: string[];
table?: string[][];
notes?: string[];
}
const P_NS =
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
function aParagraph(text: string, level = 0): string {
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
}
function shape(id: number, placeholder: string | null, paragraphs: string): string {
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
}
function slideXml(slide: PptxSlide): string {
const shapes: string[] = [];
let id = 2;
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
if (slide.bullets) {
shapes.push(
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
);
}
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
if (slide.table) {
const rows = slide.table
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
.join('');
shapes.push(
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
);
}
// Every slide also carries a slide-number placeholder, which must be ignored.
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
}
function notesXml(lines: string[]): string {
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
2,
'sldImg',
'',
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
}
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
),
'docProps/core.xml': strToU8(coreProps(options.title)),
'ppt/presentation.xml': strToU8(
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
.join('')}</p:sldIdLst></p:presentation>`,
),
'ppt/_rels/presentation.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
.join('')}</Relationships>`,
),
};
for (const slide of slides) {
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
if (slide.notes) {
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
);
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
}
}
return zipSync(files);
}
// ---------------------------------------------------------------------------
// Misc
// ---------------------------------------------------------------------------
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
export const TINY_PNG = Uint8Array.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
]);
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
for (let i = 0; i + 46 < zip.length; i++) {
if (view.getUint32(i, true) !== 0x02014b50) continue;
const nameLen = view.getUint16(i + 28, true);
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
if (entryName === name) view.setUint32(i + 24, 16, true);
}
return zip;
}

53
server/documents/types.ts Normal file
View file

@ -0,0 +1,53 @@
/** Contract for the document extraction + chunking pipeline. */
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
export interface ExtractedSection {
/** Plain text (Markdown allowed) for this page / slide / section. */
text: string;
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
page?: number;
/** Nearest heading or slide title, when known. */
heading?: string;
}
export interface ExtractResult {
kind: DocKind;
sections: ExtractedSection[];
/** Page or slide count for paged formats. */
pageCount?: number;
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
warnings: string[];
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
needsOcr: boolean;
/** Document title from metadata, when present. */
title?: string;
}
export interface DocChunk {
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
}
export interface ChunkOptions {
/** Preferred chunk size in estimated tokens. */
targetTokens?: number;
/** Hard ceiling; a chunk is split before exceeding it. */
maxTokens?: number;
/** Tokens of trailing context repeated at the start of the next chunk. */
overlapTokens?: number;
}
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
export class ExtractionError extends Error {
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
super(message, opts);
this.name = 'ExtractionError';
this.code = code;
}
}

213
server/documents/zip.ts Normal file
View file

@ -0,0 +1,213 @@
/**
* Minimal, defensive ZIP reader for OOXML packages (DOCX / PPTX).
*
* Reads the central directory itself and inflates entries through fflate's
* streaming inflater in small input slices, so both memory and CPU are bounded
* by the limits below even when the archive lies about its sizes (zip bombs).
*/
import { Inflate } from 'fflate';
export interface ZipEntry {
name: string;
method: number;
compressedSize: number;
size: number;
offset: number;
encrypted: boolean;
}
export interface ZipLimits {
/** Largest uncompressed size accepted for a single entry. */
maxEntryBytes: number;
/** Largest total of uncompressed bytes read through one reader. */
maxTotalBytes: number;
}
export class ZipError extends Error {
code: 'corrupt' | 'too_large';
constructor(code: ZipError['code'], message: string) {
super(message);
this.name = 'ZipError';
this.code = code;
}
}
const EOCD_SIG = 0x06054b50;
const ZIP64_LOCATOR_SIG = 0x07064b50;
const ZIP64_EOCD_SIG = 0x06064b50;
const CENTRAL_SIG = 0x02014b50;
const LOCAL_SIG = 0x04034b50;
const MAX_ENTRIES = 20_000;
const INFLATE_STEP = 64 * 1024;
const nameDecoder = new TextDecoder('utf-8');
function u16(b: Uint8Array, o: number): number {
return b[o] | (b[o + 1] << 8);
}
function u32(b: Uint8Array, o: number): number {
return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
}
function u64(b: Uint8Array, o: number): number {
return u32(b, o) + u32(b, o + 4) * 0x1_0000_0000;
}
/** Parses the central directory. Throws ZipError('corrupt') when it cannot be read. */
export function listZip(bytes: Uint8Array): ZipEntry[] {
const len = bytes.length;
if (len < 22) throw new ZipError('corrupt', 'Not a zip archive');
let eocd = -1;
const stop = Math.max(0, len - 22 - 0xffff);
for (let i = len - 22; i >= stop; i--) {
if (bytes[i] === 0x50 && u32(bytes, i) === EOCD_SIG) {
eocd = i;
break;
}
}
if (eocd < 0) throw new ZipError('corrupt', 'Zip directory not found');
let count = u16(bytes, eocd + 10);
let dirOffset = u32(bytes, eocd + 16);
if (count === 0xffff || dirOffset === 0xffffffff) {
const locator = eocd - 20;
if (locator < 0 || u32(bytes, locator) !== ZIP64_LOCATOR_SIG) throw new ZipError('corrupt', 'Bad zip64 locator');
const z = u64(bytes, locator + 8);
if (z + 56 > len || u32(bytes, z) !== ZIP64_EOCD_SIG) throw new ZipError('corrupt', 'Bad zip64 directory');
count = u64(bytes, z + 32);
dirOffset = u64(bytes, z + 48);
}
if (count > MAX_ENTRIES) throw new ZipError('too_large', 'Too many files in archive');
const entries: ZipEntry[] = [];
let p = dirOffset;
for (let i = 0; i < count; i++) {
if (p + 46 > len || u32(bytes, p) !== CENTRAL_SIG) throw new ZipError('corrupt', 'Bad zip directory entry');
const flags = u16(bytes, p + 8);
const method = u16(bytes, p + 10);
let compressedSize = u32(bytes, p + 20);
let size = u32(bytes, p + 24);
const nameLen = u16(bytes, p + 28);
const extraLen = u16(bytes, p + 30);
const commentLen = u16(bytes, p + 32);
let offset = u32(bytes, p + 42);
const nameStart = p + 46;
const extraStart = nameStart + nameLen;
const next = extraStart + extraLen + commentLen;
if (next > len) throw new ZipError('corrupt', 'Truncated zip directory');
if (size === 0xffffffff || compressedSize === 0xffffffff || offset === 0xffffffff) {
// zip64 extended information: only the saturated fields are present, in this order.
let e = extraStart;
const extraEnd = extraStart + extraLen;
while (e + 4 <= extraEnd) {
const id = u16(bytes, e);
const dataLen = u16(bytes, e + 2);
if (id === 1) {
let q = e + 4;
if (size === 0xffffffff && q + 8 <= extraEnd) (size = u64(bytes, q)), (q += 8);
if (compressedSize === 0xffffffff && q + 8 <= extraEnd) (compressedSize = u64(bytes, q)), (q += 8);
if (offset === 0xffffffff && q + 8 <= extraEnd) offset = u64(bytes, q);
break;
}
e += 4 + dataLen;
}
}
const name = nameDecoder.decode(bytes.subarray(nameStart, extraStart)).replace(/\\/g, '/');
entries.push({ name, method, compressedSize, size, offset, encrypted: (flags & 1) === 1 });
p = next;
}
return entries;
}
/** Random-access reader over a zip held in memory, with per-entry and total size limits. */
export class ZipReader {
private readonly bytes: Uint8Array;
private readonly limits: ZipLimits;
private readonly byName = new Map<string, ZipEntry>();
private totalRead = 0;
readonly entries: ZipEntry[];
constructor(bytes: Uint8Array, limits: ZipLimits) {
this.bytes = bytes;
this.limits = limits;
this.entries = listZip(bytes);
for (const entry of this.entries) if (!this.byName.has(entry.name)) this.byName.set(entry.name, entry);
}
has(name: string): boolean {
return this.byName.has(name);
}
names(): string[] {
return this.entries.map((entry) => entry.name);
}
/** Uncompressed bytes of an entry, or null when it does not exist. */
read(name: string): Uint8Array | null {
const entry = this.byName.get(name);
if (!entry) return null;
if (entry.encrypted) throw new ZipError('corrupt', 'Encrypted zip entry');
if (entry.size > this.limits.maxEntryBytes) throw new ZipError('too_large', `Entry too large: ${name}`);
if (this.totalRead + entry.size > this.limits.maxTotalBytes) throw new ZipError('too_large', 'Archive too large');
const b = this.bytes;
const o = entry.offset;
if (o + 30 > b.length || u32(b, o) !== LOCAL_SIG) throw new ZipError('corrupt', 'Bad zip file header');
const dataStart = o + 30 + u16(b, o + 26) + u16(b, o + 28);
const dataEnd = dataStart + entry.compressedSize;
if (dataEnd > b.length) throw new ZipError('corrupt', 'Truncated zip entry');
const compressed = b.subarray(dataStart, dataEnd);
let out: Uint8Array;
if (entry.method === 0) {
if (compressed.length !== entry.size) throw new ZipError('corrupt', 'Zip entry size mismatch');
out = compressed;
} else if (entry.method === 8) {
out = this.inflate(compressed, entry.size);
} else {
throw new ZipError('corrupt', `Unsupported zip compression method ${entry.method}`);
}
this.totalRead += out.length;
return out;
}
readText(name: string): string | null {
const data = this.read(name);
if (!data) return null;
let text = new TextDecoder('utf-8').decode(data);
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
return text;
}
private inflate(compressed: Uint8Array, expected: number): Uint8Array {
const out = new Uint8Array(expected);
let written = 0;
let overflow = false;
const inflater = new Inflate((chunk) => {
if (overflow) return;
if (written + chunk.length > expected) {
overflow = true;
return;
}
out.set(chunk, written);
written += chunk.length;
});
try {
if (compressed.length === 0) {
inflater.push(compressed, true);
} else {
for (let p = 0; p < compressed.length && !overflow; p += INFLATE_STEP) {
const end = Math.min(p + INFLATE_STEP, compressed.length);
inflater.push(compressed.subarray(p, end), end === compressed.length);
}
}
} catch {
throw new ZipError('corrupt', 'Zip entry could not be decompressed');
}
// More data than the directory declared is the signature of a zip bomb.
if (overflow) throw new ZipError('too_large', 'Zip entry larger than declared');
if (written !== expected) throw new ZipError('corrupt', 'Zip entry size mismatch');
return out;
}
}

63
server/http.ts Normal file
View file

@ -0,0 +1,63 @@
import type { Context } from 'hono';
import { z } from 'zod';
import type { Config } from './config.ts';
import type { DB } from './db.ts';
/** Everything a route needs. Built once in main.ts (or per test). */
export interface AppContext {
db: DB;
config: Config;
log: (level: 'info' | 'warn' | 'error', message: string, extra?: unknown) => void;
}
export class HttpError extends Error {
status: number;
code: string;
details?: unknown;
constructor(status: number, code: string, message: string, details?: unknown) {
super(message);
this.name = 'HttpError';
this.status = status;
this.code = code;
this.details = details;
}
}
export const notFound = (what = 'Item') => new HttpError(404, 'not_found', `${what} not found`);
export const badRequest = (message: string, details?: unknown) => new HttpError(400, 'bad_request', message, details);
export const conflict = (message: string) => new HttpError(409, 'conflict', message);
export interface FieldIssue {
path: string;
message: string;
}
export function zodIssues(error: z.ZodError): FieldIssue[] {
return error.issues.map((i) => ({ path: i.path.join('.'), message: i.message }));
}
export function parse<S extends z.ZodType>(schema: S, data: unknown): z.output<S> {
const result = schema.safeParse(data);
if (!result.success) {
const issues = zodIssues(result.error);
const first = issues[0];
throw new HttpError(422, 'validation', first ? `${first.path ? `${first.path}: ` : ''}${first.message}` : 'Invalid input', issues);
}
return result.data;
}
export async function readBody<S extends z.ZodType>(c: Context, schema: S): Promise<z.output<S>> {
let json: unknown;
try {
json = await c.req.json();
} catch {
throw badRequest('Request body must be JSON');
}
return parse(schema, json);
}
export const idParam = (c: Context, name = 'id'): string => {
const v = c.req.param(name);
if (!v || v.length > 64) throw badRequest('Invalid id');
return v;
};

372
server/migrations.ts Normal file
View file

@ -0,0 +1,372 @@
/**
* Schema migrations. Append only: never edit a migration that has shipped.
* Timestamps are integer milliseconds since the Unix epoch. JSON lives in TEXT columns.
*/
export interface Migration {
id: number;
name: string;
sql: string;
}
const init = `
CREATE TABLE settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
CREATE TABLE sessions (
token_hash TEXT PRIMARY KEY,
created_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL,
last_seen_at INTEGER NOT NULL,
user_agent TEXT NOT NULL DEFAULT '',
ip TEXT NOT NULL DEFAULT ''
);
CREATE TABLE classes (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
code TEXT NOT NULL DEFAULT '',
description TEXT NOT NULL DEFAULT '',
color TEXT NOT NULL DEFAULT 'cobalt',
archived INTEGER NOT NULL DEFAULT 0,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE folders (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
name TEXT NOT NULL,
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX folders_class ON folders(class_id, sort_order);
CREATE TABLE quizzes (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX quizzes_class ON quizzes(class_id, sort_order);
CREATE TABLE questions (
id TEXT PRIMARY KEY,
quiz_id TEXT NOT NULL REFERENCES quizzes(id) ON DELETE CASCADE,
type TEXT NOT NULL CHECK (type IN ('single','multi')),
stem TEXT NOT NULL,
options TEXT NOT NULL,
rationale TEXT NOT NULL DEFAULT '',
topic TEXT NOT NULL DEFAULT '',
difficulty TEXT NOT NULL DEFAULT 'medium',
tags TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX questions_quiz ON questions(quiz_id, sort_order);
CREATE TABLE attempts (
id TEXT PRIMARY KEY,
quiz_id TEXT REFERENCES quizzes(id) ON DELETE SET NULL,
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
title TEXT NOT NULL,
mode TEXT NOT NULL CHECK (mode IN ('practice','exam')),
kind TEXT NOT NULL DEFAULT 'full',
status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active','completed')),
settings TEXT NOT NULL DEFAULT '{}',
snapshot TEXT NOT NULL,
answers TEXT NOT NULL DEFAULT '{}',
flags TEXT NOT NULL DEFAULT '[]',
current_index INTEGER NOT NULL DEFAULT 0,
elapsed_sec INTEGER NOT NULL DEFAULT 0,
question_count INTEGER NOT NULL DEFAULT 0,
score REAL,
max_score REAL,
started_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
completed_at INTEGER
);
CREATE INDEX attempts_quiz ON attempts(quiz_id, status, completed_at);
CREATE INDEX attempts_status ON attempts(status, updated_at);
CREATE TABLE responses (
id INTEGER PRIMARY KEY AUTOINCREMENT,
attempt_id TEXT NOT NULL REFERENCES attempts(id) ON DELETE CASCADE,
question_id TEXT NOT NULL,
quiz_id TEXT,
class_id TEXT,
topic TEXT NOT NULL DEFAULT '',
score REAL NOT NULL,
correct INTEGER NOT NULL,
answered_at INTEGER NOT NULL,
UNIQUE (attempt_id, question_id)
);
CREATE INDEX responses_question ON responses(question_id, answered_at);
CREATE INDEX responses_time ON responses(answered_at);
CREATE INDEX responses_class ON responses(class_id, answered_at);
CREATE TABLE decks (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX decks_class ON decks(class_id, sort_order);
CREATE TABLE cards (
id TEXT PRIMARY KEY,
deck_id TEXT NOT NULL REFERENCES decks(id) ON DELETE CASCADE,
front TEXT NOT NULL,
back TEXT NOT NULL,
hint TEXT NOT NULL DEFAULT '',
tags TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
suspended INTEGER NOT NULL DEFAULT 0,
state INTEGER NOT NULL DEFAULT 0,
due INTEGER,
stability REAL NOT NULL DEFAULT 0,
difficulty REAL NOT NULL DEFAULT 0,
elapsed_days INTEGER NOT NULL DEFAULT 0,
scheduled_days INTEGER NOT NULL DEFAULT 0,
learning_steps INTEGER NOT NULL DEFAULT 0,
reps INTEGER NOT NULL DEFAULT 0,
lapses INTEGER NOT NULL DEFAULT 0,
last_review INTEGER,
introduced_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX cards_deck ON cards(deck_id, sort_order);
CREATE INDEX cards_due ON cards(state, due);
CREATE TABLE reviews (
id INTEGER PRIMARY KEY AUTOINCREMENT,
card_id TEXT NOT NULL,
deck_id TEXT,
class_id TEXT,
rating INTEGER NOT NULL,
state_before INTEGER NOT NULL DEFAULT 0,
scheduled_days INTEGER NOT NULL DEFAULT 0,
mode TEXT NOT NULL DEFAULT 'srs',
prev TEXT NOT NULL DEFAULT '',
reviewed_at INTEGER NOT NULL
);
CREATE INDEX reviews_time ON reviews(reviewed_at);
CREATE INDEX reviews_card ON reviews(card_id, reviewed_at);
CREATE TABLE deck_sessions (
deck_id TEXT PRIMARY KEY REFERENCES decks(id) ON DELETE CASCADE,
state TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE notes (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
content TEXT NOT NULL DEFAULT '',
kind TEXT NOT NULL DEFAULT 'note',
source TEXT NOT NULL DEFAULT '{}',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX notes_class ON notes(class_id, sort_order);
CREATE TABLE documents (
id TEXT PRIMARY KEY,
class_id TEXT NOT NULL REFERENCES classes(id) ON DELETE CASCADE,
folder_id TEXT REFERENCES folders(id) ON DELETE SET NULL,
title TEXT NOT NULL,
filename TEXT NOT NULL,
mime TEXT NOT NULL DEFAULT '',
kind TEXT NOT NULL,
size INTEGER NOT NULL DEFAULT 0,
sha256 TEXT NOT NULL DEFAULT '',
storage_name TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
error TEXT NOT NULL DEFAULT '',
progress REAL NOT NULL DEFAULT 0,
page_count INTEGER,
char_count INTEGER NOT NULL DEFAULT 0,
token_count INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0,
needs_ocr INTEGER NOT NULL DEFAULT 0,
warnings TEXT NOT NULL DEFAULT '[]',
sort_order INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX documents_class ON documents(class_id, sort_order);
CREATE TABLE chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
class_id TEXT NOT NULL,
seq INTEGER NOT NULL,
text TEXT NOT NULL,
heading TEXT NOT NULL DEFAULT '',
page_start INTEGER,
page_end INTEGER,
tokens INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX chunks_doc ON chunks(document_id, seq);
CREATE VIRTUAL TABLE chunks_fts USING fts5(text, heading, content='chunks', content_rowid='id', tokenize='porter unicode61');
CREATE TRIGGER chunks_ai AFTER INSERT ON chunks BEGIN
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
END;
CREATE TRIGGER chunks_ad AFTER DELETE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
END;
CREATE TRIGGER chunks_au AFTER UPDATE ON chunks BEGIN
INSERT INTO chunks_fts(chunks_fts, rowid, text, heading) VALUES ('delete', old.id, old.text, old.heading);
INSERT INTO chunks_fts(rowid, text, heading) VALUES (new.id, new.text, new.heading);
END;
CREATE TABLE shares (
id TEXT PRIMARY KEY,
token TEXT NOT NULL UNIQUE,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','folder','class','note','document')),
target_id TEXT NOT NULL,
enabled INTEGER NOT NULL DEFAULT 1,
view_count INTEGER NOT NULL DEFAULT 0,
last_viewed_at INTEGER,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (kind, target_id)
);
CREATE TABLE templates (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','generation')),
name TEXT NOT NULL,
description TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL,
builtin INTEGER NOT NULL DEFAULT 0,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE media (
id TEXT PRIMARY KEY,
filename TEXT NOT NULL,
mime TEXT NOT NULL,
size INTEGER NOT NULL,
created_at INTEGER NOT NULL
);
CREATE TABLE ai_providers (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
kind TEXT NOT NULL,
base_url TEXT,
api_key_enc TEXT,
headers_enc TEXT,
enabled INTEGER NOT NULL DEFAULT 1,
last_test_at INTEGER,
last_test_ok INTEGER,
last_test_message TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE ai_models (
id TEXT PRIMARY KEY,
provider_id TEXT NOT NULL REFERENCES ai_providers(id) ON DELETE CASCADE,
model TEXT NOT NULL,
label TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT 'discovered',
enabled INTEGER NOT NULL DEFAULT 0,
capabilities TEXT NOT NULL DEFAULT '{}',
context_window INTEGER,
max_output INTEGER,
price_in REAL,
price_out REAL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
UNIQUE (provider_id, model)
);
CREATE TABLE ai_usage (
id INTEGER PRIMARY KEY AUTOINCREMENT,
at INTEGER NOT NULL,
provider_id TEXT,
provider_name TEXT NOT NULL DEFAULT '',
model TEXT NOT NULL DEFAULT '',
task TEXT NOT NULL DEFAULT '',
input_tokens INTEGER NOT NULL DEFAULT 0,
output_tokens INTEGER NOT NULL DEFAULT 0,
cache_read_tokens INTEGER NOT NULL DEFAULT 0,
cost REAL,
ok INTEGER NOT NULL DEFAULT 1,
error TEXT NOT NULL DEFAULT '',
duration_ms INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX ai_usage_at ON ai_usage(at);
CREATE TABLE conversations (
id TEXT PRIMARY KEY,
title TEXT NOT NULL DEFAULT '',
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE messages (
id INTEGER PRIMARY KEY AUTOINCREMENT,
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
role TEXT NOT NULL,
content TEXT NOT NULL,
meta TEXT NOT NULL DEFAULT '{}',
created_at INTEGER NOT NULL
);
CREATE INDEX messages_conv ON messages(conversation_id, id);
CREATE TABLE drafts (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL CHECK (kind IN ('quiz','deck','note')),
class_id TEXT REFERENCES classes(id) ON DELETE SET NULL,
folder_id TEXT,
title TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}',
meta TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'ready',
progress REAL NOT NULL DEFAULT 0,
progress_label TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
saved_kind TEXT,
saved_id TEXT,
conversation_id TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE INDEX drafts_status ON drafts(status, updated_at);
CREATE TABLE ai_actions (
id TEXT PRIMARY KEY,
conversation_id TEXT,
kind TEXT NOT NULL,
summary TEXT NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'pending',
result TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
resolved_at INTEGER
);
`;
export const MIGRATIONS: Migration[] = [{ id: 1, name: 'init', sql: init }];

564
server/routes/library.ts Normal file
View file

@ -0,0 +1,564 @@
import fs from 'node:fs';
import path from 'node:path';
import { Hono } from 'hono';
import { z } from 'zod';
import { CLASS_COLORS, deckCreateSchema, descriptionSchema, questionToDraft, quizCreateSchema, titleSchema } from '../../shared/content.ts';
import { exportDeck, exportQuiz } from '../../shared/importer.ts';
import { badRequest, idParam, notFound, readBody, type AppContext } from '../http.ts';
import { getAppSettings } from '../settings.ts';
import { listAttempts } from '../services/attempts.ts';
import {
classInputSchema,
createClass,
createDeck,
createFolder,
createNote,
createQuiz,
duplicateDeck,
duplicateQuiz,
getClass,
getDeck,
getFolder,
getNote,
getQuiz,
insertCards,
insertQuestions,
listCards,
listQuestions,
mapClass,
mapDeck,
mapFolder,
mapNote,
mapQuiz,
resolveFolder,
updateCard,
updateQuestion,
type ClassRow,
type DeckRow,
type FolderRow,
type NoteRow,
type QuizRow,
} from '../services/content.ts';
import { deckStats, dueCounts, resetCards } from '../services/srs.ts';
import { questionStats } from '../services/stats.ts';
import { now, parseJson, slugify } from '../util.ts';
function download(c: import('hono').Context, filename: string, data: unknown) {
c.header('Content-Disposition', `attachment; filename="${filename}"`);
return c.json(data);
}
export function shareFor(app: AppContext, kind: string, targetId: string) {
const row = app.db.get<{ id: string; token: string; enabled: number; view_count: number; last_viewed_at: number | null; created_at: number }>(
'SELECT * FROM shares WHERE kind = ? AND target_id = ?',
kind,
targetId,
);
return row
? { id: row.id, token: row.token, enabled: Boolean(row.enabled), viewCount: row.view_count, lastViewedAt: row.last_viewed_at, createdAt: row.created_at }
: null;
}
export function libraryRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const settings = () => getAppSettings(db, app.config.timezone);
/* ------------------------------ classes ------------------------------ */
r.get('/classes', (c) => {
const rows = db.all<ClassRow>('SELECT * FROM classes ORDER BY archived, sort_order, created_at');
const count = (table: string) =>
new Map(db.all<{ class_id: string; n: number }>(`SELECT class_id, COUNT(*) AS n FROM ${table} GROUP BY class_id`).map((x) => [x.class_id, x.n]));
const quizzes = count('quizzes');
const decks = count('decks');
const documents = count('documents');
const notes = count('notes');
const s = settings();
return c.json({
classes: rows.map((row) => ({
...mapClass(row),
counts: { quizzes: quizzes.get(row.id) ?? 0, decks: decks.get(row.id) ?? 0, documents: documents.get(row.id) ?? 0, notes: notes.get(row.id) ?? 0 },
due: row.archived ? 0 : dueCounts(db, { classId: row.id }, s).total,
})),
});
});
r.post('/classes', async (c) => {
const body = await readBody(c, classInputSchema);
return c.json({ class: mapClass(createClass(db, body)) }, 201);
});
r.post('/classes/reorder', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(500) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE classes SET sort_order = ? WHERE id = ?', i, id)));
return c.json({ ok: true });
});
r.get('/classes/:id', (c) => {
const cls = getClass(db, idParam(c));
const s = settings();
const shares = new Map(
db.all<{ kind: string; target_id: string; enabled: number }>('SELECT kind, target_id, enabled FROM shares').map((x) => [`${x.kind}:${x.target_id}`, Boolean(x.enabled)]),
);
const shared = (kind: string, id: string) => shares.get(`${kind}:${id}`) ?? false;
const quizzes = db.all<QuizRow & { qn: number; attempts: number; last_score: number | null; last_max: number | null; last_at: number | null; active_id: string | null }>(
`SELECT z.*,
(SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS qn,
(SELECT COUNT(*) FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed') AS attempts,
(SELECT a.score FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' AND a.kind = 'full' ORDER BY a.completed_at DESC LIMIT 1) AS last_score,
(SELECT a.max_score FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' AND a.kind = 'full' ORDER BY a.completed_at DESC LIMIT 1) AS last_max,
(SELECT a.completed_at FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'completed' ORDER BY a.completed_at DESC LIMIT 1) AS last_at,
(SELECT a.id FROM attempts a WHERE a.quiz_id = z.id AND a.status = 'active' ORDER BY a.updated_at DESC LIMIT 1) AS active_id
FROM quizzes z WHERE z.class_id = ?`,
cls.id,
);
const decks = db.all<DeckRow & { cn: number }>(`SELECT d.*, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS cn FROM decks d WHERE d.class_id = ?`, cls.id);
const documents = db.all<Record<string, any>>(
'SELECT id, folder_id, title, filename, kind, size, status, error, progress, page_count, token_count, needs_ocr, created_at, updated_at FROM documents WHERE class_id = ?',
cls.id,
);
const notes = db.all<NoteRow>('SELECT * FROM notes WHERE class_id = ?', cls.id);
const items = [
...quizzes.map((q) => ({
kind: 'quiz' as const,
id: q.id,
title: q.title,
description: q.description,
folderId: q.folder_id,
createdAt: q.created_at,
updatedAt: q.updated_at,
shared: shared('quiz', q.id),
questionCount: q.qn,
attempts: q.attempts,
lastPercent: q.last_max ? Math.round(((q.last_score ?? 0) / q.last_max) * 100) : null,
lastStudiedAt: q.last_at,
activeAttemptId: q.active_id,
})),
...decks.map((d) => {
const st = deckStats(db, d.id, s);
return {
kind: 'deck' as const,
id: d.id,
title: d.title,
description: d.description,
folderId: d.folder_id,
createdAt: d.created_at,
updatedAt: d.updated_at,
shared: shared('deck', d.id),
cardCount: d.cn,
dueCount: st.dueNow,
newCount: st.new,
learnedCount: st.young + st.mature,
};
}),
...documents.map((d) => ({
kind: 'document' as const,
id: d.id as string,
title: d.title as string,
description: '',
folderId: d.folder_id as string | null,
createdAt: d.created_at as number,
updatedAt: d.updated_at as number,
shared: shared('document', d.id),
docKind: d.kind as string,
filename: d.filename as string,
size: d.size as number,
status: d.status as string,
error: d.error as string,
progress: d.progress as number,
pageCount: d.page_count as number | null,
tokenCount: d.token_count as number,
needsOcr: Boolean(d.needs_ocr),
})),
...notes.map((n) => ({
kind: 'note' as const,
id: n.id,
title: n.title,
description: n.content.replace(/[#*_>`|-]/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 160),
folderId: n.folder_id,
createdAt: n.created_at,
updatedAt: n.updated_at,
shared: shared('note', n.id),
noteKind: n.kind,
})),
];
const folders = db.all<FolderRow>('SELECT * FROM folders WHERE class_id = ? ORDER BY sort_order, created_at', cls.id);
return c.json({
class: mapClass(cls),
folders: folders.map((f) => ({ ...mapFolder(f), shared: shared('folder', f.id) })),
items,
due: dueCounts(db, { classId: cls.id }, s),
share: shareFor(app, 'class', cls.id),
});
});
r.patch('/classes/:id', async (c) => {
const cls = getClass(db, idParam(c));
const body = await readBody(
c,
z.object({
name: z.string().trim().min(1).max(120).optional(),
code: z.string().trim().max(40).optional(),
description: z.string().max(2000).optional(),
color: z.enum(CLASS_COLORS).optional(),
archived: z.boolean().optional(),
}),
);
db.run(
'UPDATE classes SET name = ?, code = ?, description = ?, color = ?, archived = ?, updated_at = ? WHERE id = ?',
body.name ?? cls.name,
body.code ?? cls.code,
body.description ?? cls.description,
body.color ?? cls.color,
body.archived === undefined ? cls.archived : body.archived ? 1 : 0,
now(),
cls.id,
);
return c.json({ class: mapClass(getClass(db, cls.id)) });
});
r.delete('/classes/:id', (c) => {
const cls = getClass(db, idParam(c));
const files = db.all<{ storage_name: string }>('SELECT storage_name FROM documents WHERE class_id = ?', cls.id);
db.tx(() => {
db.run(
`DELETE FROM shares WHERE (kind = 'class' AND target_id = ?)
OR (kind = 'folder' AND target_id IN (SELECT id FROM folders WHERE class_id = ?))
OR (kind = 'quiz' AND target_id IN (SELECT id FROM quizzes WHERE class_id = ?))
OR (kind = 'deck' AND target_id IN (SELECT id FROM decks WHERE class_id = ?))
OR (kind = 'note' AND target_id IN (SELECT id FROM notes WHERE class_id = ?))
OR (kind = 'document' AND target_id IN (SELECT id FROM documents WHERE class_id = ?))`,
cls.id,
cls.id,
cls.id,
cls.id,
cls.id,
cls.id,
);
db.run('DELETE FROM classes WHERE id = ?', cls.id);
});
for (const f of files) fs.rmSync(path.join(app.config.uploadsDir, path.basename(f.storage_name)), { force: true });
return c.json({ ok: true });
});
/* ------------------------------ folders ------------------------------ */
r.post('/classes/:id/folders', async (c) => {
const { name } = await readBody(c, z.object({ name: z.string().trim().min(1).max(120) }));
return c.json({ folder: mapFolder(createFolder(db, idParam(c), name)) }, 201);
});
r.patch('/folders/:id', async (c) => {
const folder = getFolder(db, idParam(c));
const { name } = await readBody(c, z.object({ name: z.string().trim().min(1).max(120) }));
db.run('UPDATE folders SET name = ?, updated_at = ? WHERE id = ?', name, now(), folder.id);
return c.json({ folder: mapFolder(getFolder(db, folder.id)) });
});
r.delete('/folders/:id', (c) => {
const folder = getFolder(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'folder' AND target_id = ?`, folder.id);
db.run('DELETE FROM folders WHERE id = ?', folder.id);
});
return c.json({ ok: true });
});
r.post('/classes/:id/folders/reorder', async (c) => {
const classId = idParam(c);
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(500) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE folders SET sort_order = ? WHERE id = ? AND class_id = ?', i, id, classId)));
return c.json({ ok: true });
});
/* ------------------------------ move (any kind) ------------------------------ */
const TABLES = { quiz: 'quizzes', deck: 'decks', note: 'notes', document: 'documents' } as const;
r.post('/items/move', async (c) => {
const body = await readBody(
c,
z.object({
items: z.array(z.object({ kind: z.enum(['quiz', 'deck', 'note', 'document']), id: z.string() })).min(1).max(500),
classId: z.string(),
folderId: z.string().nullish(),
}),
);
getClass(db, body.classId);
const folderId = resolveFolder(db, body.classId, body.folderId);
db.tx(() => {
for (const item of body.items) {
const table = TABLES[item.kind];
db.run(`UPDATE ${table} SET class_id = ?, folder_id = ?, updated_at = ? WHERE id = ?`, body.classId, folderId, now(), item.id);
if (item.kind === 'document') db.run('UPDATE chunks SET class_id = ? WHERE document_id = ?', body.classId, item.id);
}
});
return c.json({ ok: true });
});
/* ------------------------------ quizzes ------------------------------ */
r.post('/quizzes', async (c) => {
const body = await readBody(c, quizCreateSchema);
const quiz = createQuiz(db, body);
return c.json({ quiz: mapQuiz(quiz) }, 201);
});
r.get('/quizzes/:id', (c) => {
const quiz = getQuiz(db, idParam(c));
const cls = getClass(db, quiz.class_id);
const history = listAttempts(db, { quizId: quiz.id, status: 'completed', limit: 100 });
const active = listAttempts(db, { quizId: quiz.id, status: 'active', limit: 1 })[0] ?? null;
return c.json({
quiz: mapQuiz(quiz),
class: mapClass(cls),
questions: listQuestions(db, quiz.id),
questionStats: questionStats(db, quiz.id),
history,
activeAttempt: active,
share: shareFor(app, 'quiz', quiz.id),
});
});
const itemPatch = z.object({ title: titleSchema.optional(), description: descriptionSchema.optional(), folderId: z.string().nullish() });
r.patch('/quizzes/:id', async (c) => {
const quiz = getQuiz(db, idParam(c));
const body = await readBody(c, itemPatch);
db.run(
'UPDATE quizzes SET title = ?, description = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? quiz.title,
body.description ?? quiz.description,
body.folderId === undefined ? quiz.folder_id : resolveFolder(db, quiz.class_id, body.folderId),
now(),
quiz.id,
);
return c.json({ quiz: mapQuiz(getQuiz(db, quiz.id)) });
});
r.delete('/quizzes/:id', (c) => {
const quiz = getQuiz(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'quiz' AND target_id = ?`, quiz.id);
db.run(`DELETE FROM attempts WHERE quiz_id = ? AND status = 'active'`, quiz.id);
db.run('DELETE FROM quizzes WHERE id = ?', quiz.id);
});
return c.json({ ok: true });
});
const duplicateSchema = z.object({ classId: z.string().optional(), folderId: z.string().nullish(), title: titleSchema.optional() }).optional();
r.post('/quizzes/:id/duplicate', async (c) => {
const body = await c.req.json().catch(() => ({}));
const target = duplicateSchema.parse(body) ?? {};
return c.json({ quiz: mapQuiz(duplicateQuiz(db, idParam(c), target)) }, 201);
});
r.get('/quizzes/:id/export', (c) => {
const quiz = getQuiz(db, idParam(c));
const data = exportQuiz({ title: quiz.title, description: quiz.description, questions: listQuestions(db, quiz.id).map(questionToDraft) });
return download(c, `${slugify(quiz.title)}.quiz.json`, data);
});
r.post('/quizzes/:id/questions', async (c) => {
const quiz = getQuiz(db, idParam(c));
const body = await readBody(c, z.object({ questions: z.array(z.unknown()).min(1).max(1000), index: z.number().int().min(0).optional() }));
return c.json({ questions: insertQuestions(db, quiz.id, body.questions, body.index) }, 201);
});
r.post('/quizzes/:id/questions/reorder', async (c) => {
const quiz = getQuiz(db, idParam(c));
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(2000) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE questions SET sort_order = ? WHERE id = ? AND quiz_id = ?', i, id, quiz.id)));
return c.json({ ok: true });
});
r.patch('/questions/:id', async (c) => {
const body = await c.req.json().catch(() => {
throw badRequest('Request body must be JSON');
});
return c.json({ question: updateQuestion(db, idParam(c), body) });
});
r.delete('/questions/:id', (c) => {
const { changes } = db.run('DELETE FROM questions WHERE id = ?', idParam(c));
if (!changes) throw notFound('Question');
return c.json({ ok: true });
});
r.post('/questions/delete', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).min(1).max(2000) }));
db.tx(() => ids.forEach((id) => db.run('DELETE FROM questions WHERE id = ?', id)));
return c.json({ ok: true });
});
/* ------------------------------ decks ------------------------------ */
r.post('/decks', async (c) => {
const body = await readBody(c, deckCreateSchema);
return c.json({ deck: mapDeck(createDeck(db, body)) }, 201);
});
r.get('/decks/:id', (c) => {
const deck = getDeck(db, idParam(c));
const session = db.get<{ state: string; updated_at: number }>('SELECT state, updated_at FROM deck_sessions WHERE deck_id = ?', deck.id);
return c.json({
deck: mapDeck(deck),
class: mapClass(getClass(db, deck.class_id)),
cards: listCards(db, deck.id),
stats: deckStats(db, deck.id, settings()),
due: dueCounts(db, { deckId: deck.id }, settings()),
session: session ? { state: parseJson<unknown>(session.state, null), updatedAt: session.updated_at } : null,
share: shareFor(app, 'deck', deck.id),
});
});
r.patch('/decks/:id', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, itemPatch);
db.run(
'UPDATE decks SET title = ?, description = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? deck.title,
body.description ?? deck.description,
body.folderId === undefined ? deck.folder_id : resolveFolder(db, deck.class_id, body.folderId),
now(),
deck.id,
);
return c.json({ deck: mapDeck(getDeck(db, deck.id)) });
});
r.delete('/decks/:id', (c) => {
const deck = getDeck(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'deck' AND target_id = ?`, deck.id);
db.run('DELETE FROM decks WHERE id = ?', deck.id);
});
return c.json({ ok: true });
});
r.post('/decks/:id/duplicate', async (c) => {
const body = await c.req.json().catch(() => ({}));
const target = duplicateSchema.parse(body) ?? {};
return c.json({ deck: mapDeck(duplicateDeck(db, idParam(c), target)) }, 201);
});
r.get('/decks/:id/export', (c) => {
const deck = getDeck(db, idParam(c));
const data = exportDeck({ title: deck.title, description: deck.description, cards: listCards(db, deck.id).map((x) => ({ front: x.front, back: x.back, hint: x.hint, tags: x.tags })) });
return download(c, `${slugify(deck.title)}.deck.json`, data);
});
r.post('/decks/:id/cards', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, z.object({ cards: z.array(z.unknown()).min(1).max(5000) }));
return c.json({ cards: insertCards(db, deck.id, body.cards) }, 201);
});
r.post('/decks/:id/cards/reorder', async (c) => {
const deck = getDeck(db, idParam(c));
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).max(10000) }));
db.tx(() => ids.forEach((id, i) => db.run('UPDATE cards SET sort_order = ? WHERE id = ? AND deck_id = ?', i, id, deck.id)));
return c.json({ ok: true });
});
r.post('/decks/:id/reset', (c) => {
const deck = getDeck(db, idParam(c));
return c.json({ reset: resetCards(db, { deckId: deck.id }) });
});
r.put('/decks/:id/session', async (c) => {
const deck = getDeck(db, idParam(c));
const body = await readBody(c, z.object({ state: z.unknown() }));
const json = JSON.stringify(body.state ?? null);
if (json.length > 2_000_000) throw badRequest('Session state is too large');
db.run(
'INSERT INTO deck_sessions (deck_id, state, updated_at) VALUES (?, ?, ?) ON CONFLICT(deck_id) DO UPDATE SET state = excluded.state, updated_at = excluded.updated_at',
deck.id,
json,
now(),
);
return c.json({ ok: true });
});
r.delete('/decks/:id/session', (c) => {
db.run('DELETE FROM deck_sessions WHERE deck_id = ?', idParam(c));
return c.json({ ok: true });
});
r.patch('/cards/:id', async (c) => {
const id = idParam(c);
const body = (await c.req.json().catch(() => {
throw badRequest('Request body must be JSON');
})) as Record<string, unknown>;
if (typeof body.suspended === 'boolean') {
const { changes } = db.run('UPDATE cards SET suspended = ?, updated_at = ? WHERE id = ?', body.suspended ? 1 : 0, now(), id);
if (!changes) throw notFound('Card');
}
if (body.reset === true) resetCards(db, { cardId: id });
if (body.front !== undefined || body.back !== undefined) return c.json({ card: updateCard(db, id, body) });
const row = db.get<{ deck_id: string }>('SELECT deck_id FROM cards WHERE id = ?', id);
if (!row) throw notFound('Card');
return c.json({ card: listCards(db, row.deck_id).find((x) => x.id === id) });
});
r.delete('/cards/:id', (c) => {
const { changes } = db.run('DELETE FROM cards WHERE id = ?', idParam(c));
if (!changes) throw notFound('Card');
return c.json({ ok: true });
});
r.post('/cards/delete', async (c) => {
const { ids } = await readBody(c, z.object({ ids: z.array(z.string()).min(1).max(10000) }));
db.tx(() => ids.forEach((id) => db.run('DELETE FROM cards WHERE id = ?', id)));
return c.json({ ok: true });
});
/* ------------------------------ notes ------------------------------ */
r.post('/notes', async (c) => {
const body = await readBody(
c,
z.object({
classId: z.string(),
folderId: z.string().nullish(),
title: titleSchema,
content: z.string().max(400000).optional().default(''),
kind: z.enum(['note', 'guide', 'summary']).optional().default('note'),
}),
);
return c.json({ note: mapNote(createNote(db, body)) }, 201);
});
r.get('/notes/:id', (c) => {
const note = getNote(db, idParam(c));
return c.json({ note: mapNote(note), class: mapClass(getClass(db, note.class_id)), share: shareFor(app, 'note', note.id) });
});
r.patch('/notes/:id', async (c) => {
const note = getNote(db, idParam(c));
const body = await readBody(c, z.object({ title: titleSchema.optional(), content: z.string().max(400000).optional(), folderId: z.string().nullish() }));
db.run(
'UPDATE notes SET title = ?, content = ?, folder_id = ?, updated_at = ? WHERE id = ?',
body.title ?? note.title,
body.content ?? note.content,
body.folderId === undefined ? note.folder_id : resolveFolder(db, note.class_id, body.folderId),
now(),
note.id,
);
return c.json({ note: mapNote(getNote(db, note.id)) });
});
r.delete('/notes/:id', (c) => {
const note = getNote(db, idParam(c));
db.tx(() => {
db.run(`DELETE FROM shares WHERE kind = 'note' AND target_id = ?`, note.id);
db.run('DELETE FROM notes WHERE id = ?', note.id);
});
return c.json({ ok: true });
});
return r;
}

260
server/routes/shares.ts Normal file
View file

@ -0,0 +1,260 @@
import fs from 'node:fs';
import path from 'node:path';
import { Hono } from 'hono';
import { z } from 'zod';
import { HttpError, idParam, notFound, readBody, type AppContext } from '../http.ts';
import { listCards, listQuestions } from '../services/content.ts';
import { newId, newToken, now } from '../util.ts';
const KINDS = ['quiz', 'deck', 'folder', 'class', 'note', 'document'] as const;
type ShareKind = (typeof KINDS)[number];
interface ShareRow {
id: string;
token: string;
kind: ShareKind;
target_id: string;
enabled: number;
view_count: number;
last_viewed_at: number | null;
created_at: number;
updated_at: number;
}
const TABLE: Record<ShareKind, { table: string; title: string }> = {
quiz: { table: 'quizzes', title: 'title' },
deck: { table: 'decks', title: 'title' },
note: { table: 'notes', title: 'title' },
document: { table: 'documents', title: 'title' },
folder: { table: 'folders', title: 'name' },
class: { table: 'classes', title: 'name' },
};
function mapShare(row: ShareRow) {
return {
id: row.id,
token: row.token,
kind: row.kind,
targetId: row.target_id,
enabled: Boolean(row.enabled),
viewCount: row.view_count,
lastViewedAt: row.last_viewed_at,
createdAt: row.created_at,
};
}
/* ------------------------------ admin ------------------------------ */
export function shareRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
r.get('/shares', (c) => {
const rows = db.all<ShareRow>('SELECT * FROM shares ORDER BY created_at DESC');
const out = [];
for (const row of rows) {
const meta = TABLE[row.kind];
const target = db.get<{ title: string; class_id?: string }>(
`SELECT ${meta.title} AS title${row.kind === 'class' ? '' : ', class_id'} FROM ${meta.table} WHERE id = ?`,
row.target_id,
);
if (!target) {
db.run('DELETE FROM shares WHERE id = ?', row.id);
continue;
}
const classId = row.kind === 'class' ? row.target_id : target.class_id;
const cls = classId ? db.get<{ name: string; color: string }>('SELECT name, color FROM classes WHERE id = ?', classId) : undefined;
out.push({ ...mapShare(row), title: target.title, classId: classId ?? null, className: cls?.name ?? '', classColor: cls?.color ?? 'slate' });
}
return c.json({ shares: out });
});
r.post('/shares', async (c) => {
const body = await readBody(c, z.object({ kind: z.enum(KINDS), targetId: z.string() }));
const meta = TABLE[body.kind];
if (!db.get(`SELECT id FROM ${meta.table} WHERE id = ?`, body.targetId)) throw notFound('Item');
const existing = db.get<ShareRow>('SELECT * FROM shares WHERE kind = ? AND target_id = ?', body.kind, body.targetId);
const at = now();
if (existing) {
db.run('UPDATE shares SET enabled = 1, updated_at = ? WHERE id = ?', at, existing.id);
return c.json({ share: mapShare({ ...existing, enabled: 1 }) });
}
const id = newId();
db.run('INSERT INTO shares (id, token, kind, target_id, enabled, created_at, updated_at) VALUES (?, ?, ?, ?, 1, ?, ?)', id, newToken(), body.kind, body.targetId, at, at);
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) }, 201);
});
r.patch('/shares/:id', async (c) => {
const id = idParam(c);
const body = await readBody(c, z.object({ enabled: z.boolean() }));
const { changes } = db.run('UPDATE shares SET enabled = ?, updated_at = ? WHERE id = ?', body.enabled ? 1 : 0, now(), id);
if (!changes) throw notFound('Share link');
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) });
});
/** Issues a new token: the old link stops working immediately. */
r.post('/shares/:id/rotate', (c) => {
const id = idParam(c);
const { changes } = db.run('UPDATE shares SET token = ?, view_count = 0, last_viewed_at = NULL, updated_at = ? WHERE id = ?', newToken(), now(), id);
if (!changes) throw notFound('Share link');
return c.json({ share: mapShare(db.get<ShareRow>('SELECT * FROM shares WHERE id = ?', id)!) });
});
r.delete('/shares/:id', (c) => {
const { changes } = db.run('DELETE FROM shares WHERE id = ?', idParam(c));
if (!changes) throw notFound('Share link');
return c.json({ ok: true });
});
return r;
}
/* ------------------------------ public (no auth) ------------------------------ */
const gone = () => new HttpError(404, 'share_not_found', 'This link is no longer available');
function publicQuiz(app: AppContext, id: string) {
const quiz = app.db.get<{ id: string; title: string; description: string; class_id: string }>('SELECT id, title, description, class_id FROM quizzes WHERE id = ?', id);
if (!quiz) throw gone();
return {
kind: 'quiz' as const,
id: quiz.id,
title: quiz.title,
description: quiz.description,
questions: listQuestions(app.db, quiz.id).map((q) => ({
id: q.id,
type: q.type,
stem: q.stem,
options: q.options,
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
})),
};
}
function publicDeck(app: AppContext, id: string) {
const deck = app.db.get<{ id: string; title: string; description: string }>('SELECT id, title, description FROM decks WHERE id = ?', id);
if (!deck) throw gone();
return {
kind: 'deck' as const,
id: deck.id,
title: deck.title,
description: deck.description,
cards: listCards(app.db, deck.id).map((c) => ({ id: c.id, front: c.front, back: c.back, hint: c.hint })),
};
}
function publicNote(app: AppContext, id: string) {
const note = app.db.get<{ id: string; title: string; content: string; updated_at: number }>('SELECT id, title, content, updated_at FROM notes WHERE id = ?', id);
if (!note) throw gone();
return { kind: 'note' as const, id: note.id, title: note.title, content: note.content, updatedAt: note.updated_at };
}
function publicDocument(app: AppContext, id: string) {
const doc = app.db.get<Record<string, any>>('SELECT id, title, filename, kind, size, page_count, mime FROM documents WHERE id = ?', id);
if (!doc) throw gone();
return { kind: 'document' as const, id: doc.id as string, title: doc.title as string, filename: doc.filename as string, docKind: doc.kind as string, size: doc.size as number, pageCount: doc.page_count as number | null };
}
/** Quizzes, decks and notes inside a class or folder. Uploaded documents are never included in a collection share. */
function collectionItems(app: AppContext, share: ShareRow) {
const { db } = app;
const byFolder = share.kind === 'folder';
const where = byFolder ? 'folder_id = ?' : 'class_id = ?';
const quizzes = db.all<Record<string, any>>(
`SELECT z.id, z.title, z.description, z.folder_id, (SELECT COUNT(*) FROM questions q WHERE q.quiz_id = z.id) AS n FROM quizzes z WHERE z.${where} ORDER BY z.sort_order, z.created_at`,
share.target_id,
);
const decks = db.all<Record<string, any>>(
`SELECT d.id, d.title, d.description, d.folder_id, (SELECT COUNT(*) FROM cards c WHERE c.deck_id = d.id) AS n FROM decks d WHERE d.${where} ORDER BY d.sort_order, d.created_at`,
share.target_id,
);
const notes = db.all<Record<string, any>>(`SELECT id, title, folder_id FROM notes WHERE ${where} ORDER BY sort_order, created_at`, share.target_id);
return [
...quizzes.filter((q) => q.n > 0).map((q) => ({ kind: 'quiz' as const, id: q.id as string, title: q.title as string, description: q.description as string, folderId: q.folder_id as string | null, count: q.n as number })),
...decks.filter((d) => d.n > 0).map((d) => ({ kind: 'deck' as const, id: d.id as string, title: d.title as string, description: d.description as string, folderId: d.folder_id as string | null, count: d.n as number })),
...notes.map((n) => ({ kind: 'note' as const, id: n.id as string, title: n.title as string, description: '', folderId: n.folder_id as string | null, count: 0 })),
];
}
export function publicRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const load = (token: string): ShareRow => {
if (!token || token.length < 16 || token.length > 64) throw gone();
const share = db.get<ShareRow>('SELECT * FROM shares WHERE token = ?', token);
if (!share || !share.enabled) throw gone();
return share;
};
r.get('/:token', (c) => {
const share = load(c.req.param('token'));
db.run('UPDATE shares SET view_count = view_count + 1, last_viewed_at = ? WHERE id = ?', now(), share.id);
c.header('Cache-Control', 'no-store');
c.header('X-Robots-Tag', 'noindex, nofollow');
switch (share.kind) {
case 'quiz':
return c.json(publicQuiz(app, share.target_id));
case 'deck':
return c.json(publicDeck(app, share.target_id));
case 'note':
return c.json(publicNote(app, share.target_id));
case 'document':
return c.json(publicDocument(app, share.target_id));
case 'folder': {
const folder = db.get<{ id: string; name: string; class_id: string }>('SELECT id, name, class_id FROM folders WHERE id = ?', share.target_id);
if (!folder) throw gone();
const cls = db.get<{ name: string; color: string }>('SELECT name, color FROM classes WHERE id = ?', folder.class_id);
return c.json({ kind: 'collection' as const, title: folder.name, subtitle: cls?.name ?? '', color: cls?.color ?? 'cobalt', folders: [], items: collectionItems(app, share) });
}
case 'class': {
const cls = db.get<{ id: string; name: string; code: string; description: string; color: string }>('SELECT * FROM classes WHERE id = ?', share.target_id);
if (!cls) throw gone();
const folders = db.all<{ id: string; name: string }>('SELECT id, name FROM folders WHERE class_id = ? ORDER BY sort_order, created_at', cls.id);
return c.json({ kind: 'collection' as const, title: cls.name, subtitle: cls.code, description: cls.description, color: cls.color, folders, items: collectionItems(app, share) });
}
default:
throw gone();
}
});
/** One item inside a shared class or folder. Membership is re-checked on every request. */
r.get('/:token/items/:kind/:id', (c) => {
const share = load(c.req.param('token'));
if (share.kind !== 'class' && share.kind !== 'folder') throw gone();
const kind = c.req.param('kind');
const id = c.req.param('id');
const allowed = collectionItems(app, share).some((i) => i.kind === kind && i.id === id);
if (!allowed) throw gone();
c.header('Cache-Control', 'no-store');
c.header('X-Robots-Tag', 'noindex, nofollow');
if (kind === 'quiz') return c.json(publicQuiz(app, id));
if (kind === 'deck') return c.json(publicDeck(app, id));
return c.json(publicNote(app, id));
});
/** Download for an explicitly shared document. */
r.get('/:token/file', (c) => {
const share = load(c.req.param('token'));
if (share.kind !== 'document') throw gone();
const doc = db.get<{ filename: string; mime: string; storage_name: string }>('SELECT filename, mime, storage_name FROM documents WHERE id = ?', share.target_id);
if (!doc) throw gone();
const file = path.join(app.config.uploadsDir, path.basename(doc.storage_name));
if (!fs.existsSync(file)) throw gone();
const inline = doc.mime === 'application/pdf' || doc.mime.startsWith('image/');
return new Response(fs.readFileSync(file), {
headers: {
'Content-Type': doc.mime || 'application/octet-stream',
'Content-Disposition': `${inline ? 'inline' : 'attachment'}; filename*=UTF-8''${encodeURIComponent(doc.filename)}`,
'X-Content-Type-Options': 'nosniff',
'Content-Security-Policy': "default-src 'none'; style-src 'unsafe-inline'; sandbox",
'Cache-Control': 'private, no-store',
'X-Robots-Tag': 'noindex, nofollow',
},
});
});
return r;
}

201
server/routes/study.ts Normal file
View file

@ -0,0 +1,201 @@
import { Hono } from 'hono';
import { z } from 'zod';
import { badRequest, idParam, readBody, type AppContext } from '../http.ts';
import { getAppSettings } from '../settings.ts';
import {
attemptCreateSchema,
attemptPatchSchema,
completeAttempt,
createAttempt,
deleteAttempt,
getAttempt,
listAttempts,
missedQuestionIds,
patchAttempt,
} from '../services/attempts.ts';
import { answerCard, buildQueue, dueCounts, forecast, logCram, undoReview, type RatingValue } from '../services/srs.ts';
import { accuracyTrend, activityByDay, computeStreak, topicStats, totals, weakTopics } from '../services/stats.ts';
import { ftsQuery } from '../util.ts';
export function studyRoutes(app: AppContext): Hono {
const { db } = app;
const r = new Hono();
const settings = () => getAppSettings(db, app.config.timezone);
/* ------------------------------ quiz sessions ------------------------------ */
r.post('/attempts', async (c) => {
const body = await readBody(c, attemptCreateSchema);
const id = createAttempt(db, body, settings().sataScoring);
return c.json({ attempt: getAttempt(db, id) }, 201);
});
r.get('/attempts', (c) => {
const status = c.req.query('status');
return c.json({
attempts: listAttempts(db, {
status: status === 'active' || status === 'completed' ? status : undefined,
quizId: c.req.query('quizId') || undefined,
classId: c.req.query('classId') || undefined,
limit: Number(c.req.query('limit') ?? 50) || 50,
}),
});
});
r.get('/attempts/:id', (c) => c.json({ attempt: getAttempt(db, idParam(c)) }));
r.patch('/attempts/:id', async (c) => {
const body = await readBody(c, attemptPatchSchema);
return c.json({ attempt: patchAttempt(db, idParam(c), body) });
});
r.post('/attempts/:id/complete', async (c) => {
const body = (await c.req.json().catch(() => ({}))) as { elapsedSec?: unknown };
const elapsed = typeof body.elapsedSec === 'number' && Number.isFinite(body.elapsedSec) ? Math.max(0, Math.round(body.elapsedSec)) : undefined;
return c.json({ attempt: completeAttempt(db, idParam(c), elapsed) });
});
r.delete('/attempts/:id', (c) => {
deleteAttempt(db, idParam(c));
return c.json({ ok: true });
});
/* ------------------------------ spaced repetition ------------------------------ */
r.get('/review/queue', (c) => {
const scope = { deckId: c.req.query('deckId') || undefined, classId: c.req.query('classId') || undefined };
const limit = Math.min(Number(c.req.query('limit') ?? 100) || 100, 300);
return c.json(buildQueue(db, scope, settings(), limit));
});
r.post('/review/answer', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string(), rating: z.number().int().min(1).max(4) }));
return c.json({ result: answerCard(db, body.cardId, body.rating as RatingValue, settings()) });
});
r.post('/review/undo', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string() }));
return c.json({ result: undoReview(db, body.cardId, settings()) });
});
r.post('/review/cram', async (c) => {
const body = await readBody(c, z.object({ cardId: z.string(), known: z.boolean() }));
logCram(db, body.cardId, body.known);
return c.json({ ok: true });
});
/* ------------------------------ dashboards ------------------------------ */
r.get('/overview', (c) => {
const s = settings();
const days = activityByDay(db, s, 26 * 7);
return c.json({
settings: s,
due: dueCounts(db, {}, s),
streak: computeStreak(days, s.dailyGoal),
activity: days,
activeAttempts: listAttempts(db, { status: 'active', limit: 6 }),
recentAttempts: listAttempts(db, { status: 'completed', limit: 6 }),
weakTopics: weakTopics(db, {}, 6),
missedCount: missedQuestionIds(db, {}, 500).length,
totals: totals(db, s, {}, 7),
forecast: forecast(db, {}, s, 7),
});
});
r.get('/insights', (c) => {
const s = settings();
const classId = c.req.query('classId') || undefined;
const range = Math.min(Math.max(Number(c.req.query('days') ?? 30) || 30, 7), 365);
const scope = { classId };
const days = activityByDay(db, s, 53 * 7, scope);
return c.json({
range,
totals: totals(db, s, scope, range),
streak: computeStreak(activityByDay(db, s, 53 * 7), s.dailyGoal),
activity: days,
accuracy: accuracyTrend(db, s, range, scope),
topics: topicStats(db, scope, range),
forecast: forecast(db, scope, s, 14),
due: dueCounts(db, scope, s),
missedCount: missedQuestionIds(db, scope, 500).length,
recentAttempts: listAttempts(db, { status: 'completed', classId, limit: 15 }),
});
});
/* ------------------------------ search ------------------------------ */
r.get('/search', (c) => {
const q = (c.req.query('q') ?? '').trim().slice(0, 120);
if (q.length < 2) return c.json({ query: q, results: [] });
const like = `%${q.replace(/[\\%_]/g, (m) => `\\${m}`)}%`;
const classId = c.req.query('classId') || null;
const scoped = (col: string) => (classId ? `AND ${col} = ?` : '');
const p = (...base: string[]) => (classId ? [...base, classId] : base);
type Hit = { kind: string; id: string; title: string; snippet: string; classId: string; className: string; classColor: string; parentId?: string; page?: number | null };
const results: Hit[] = [];
for (const row of db.all<Record<string, string>>(`SELECT id, name, color FROM classes WHERE name LIKE ? ESCAPE '\\' OR code LIKE ? ESCAPE '\\' LIMIT 5`, like, like)) {
results.push({ kind: 'class', id: row.id, title: row.name, snippet: '', classId: row.id, className: row.name, classColor: row.color });
}
const simple = (kind: string, table: string) => {
for (const row of db.all<Record<string, string>>(
`SELECT t.id, t.title, k.id AS class_id, k.name AS class_name, k.color AS class_color FROM ${table} t JOIN classes k ON k.id = t.class_id
WHERE t.title LIKE ? ESCAPE '\\' ${scoped('t.class_id')} ORDER BY t.updated_at DESC LIMIT 8`,
...p(like),
)) {
results.push({ kind, id: row.id, title: row.title, snippet: '', classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
};
simple('quiz', 'quizzes');
simple('deck', 'decks');
simple('note', 'notes');
simple('document', 'documents');
for (const row of db.all<Record<string, string>>(
`SELECT q.id, q.stem, z.id AS quiz_id, z.title AS quiz_title, k.id AS class_id, k.name AS class_name, k.color AS class_color
FROM questions q JOIN quizzes z ON z.id = q.quiz_id JOIN classes k ON k.id = z.class_id
WHERE (q.stem LIKE ? ESCAPE '\\' OR q.topic LIKE ? ESCAPE '\\') ${scoped('z.class_id')} LIMIT 10`,
...p(like, like),
)) {
results.push({ kind: 'question', id: row.id, parentId: row.quiz_id, title: row.stem.slice(0, 140), snippet: row.quiz_title, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
for (const row of db.all<Record<string, string>>(
`SELECT c.id, c.front, c.back, d.id AS deck_id, d.title AS deck_title, k.id AS class_id, k.name AS class_name, k.color AS class_color
FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id
WHERE (c.front LIKE ? ESCAPE '\\' OR c.back LIKE ? ESCAPE '\\') ${scoped('d.class_id')} LIMIT 10`,
...p(like, like),
)) {
results.push({ kind: 'card', id: row.id, parentId: row.deck_id, title: row.front.slice(0, 140), snippet: row.deck_title, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
const match = ftsQuery(q, 'and');
if (match) {
try {
for (const row of db.all<Record<string, any>>(
`SELECT ch.id, ch.page_start, d.id AS doc_id, d.title AS doc_title, k.id AS class_id, k.name AS class_name, k.color AS class_color,
snippet(chunks_fts, 0, '', '', ' … ', 18) AS snip
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id JOIN classes k ON k.id = d.class_id
WHERE chunks_fts MATCH ? ${scoped('d.class_id')} ORDER BY bm25(chunks_fts) LIMIT 10`,
...p(match),
)) {
results.push({ kind: 'passage', id: String(row.id), parentId: row.doc_id, page: row.page_start, title: row.doc_title, snippet: row.snip, classId: row.class_id, className: row.class_name, classColor: row.class_color });
}
} catch {
/* a malformed FTS query should never break search */
}
}
return c.json({ query: q, results });
});
r.get('/topics', (c) => {
const classId = c.req.query('classId');
if (!classId) throw badRequest('classId is required');
const rows = db.all<{ topic: string; n: number }>(
`SELECT q.topic AS topic, COUNT(*) AS n FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE z.class_id = ? AND q.topic != '' GROUP BY q.topic ORDER BY n DESC`,
classId,
);
return c.json({ topics: rows });
});
return r;
}

431
server/services/attempts.ts Normal file
View file

@ -0,0 +1,431 @@
/**
* Quiz sessions. An attempt freezes its questions (and their shuffled answer order) in a
* snapshot when it starts, so editing or deleting the quiz later can never corrupt a session
* in progress or the review of a finished one.
*/
import { z } from 'zod';
import { isFullyCorrect, scoreQuestion, seededShuffle, type Option, type QuestionType, type SataScoring } from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
import { getClass, getQuiz, mapQuestion, type QuestionRow } from './content.ts';
export interface SnapshotQuestion {
id: string;
quizId: string | null;
quizTitle: string;
type: QuestionType;
stem: string;
options: Option[];
rationale: string;
topic: string;
difficulty: string;
}
export interface AnswerState {
selected: string[];
/** Practice mode: the answer has been checked and feedback shown. */
checked: boolean;
at: number;
}
export interface AttemptSettings {
shuffleQuestions: boolean;
shuffleOptions: boolean;
timeLimitSec: number | null;
sataScoring: SataScoring;
}
interface AttemptRow {
id: string;
quiz_id: string | null;
class_id: string | null;
title: string;
mode: 'practice' | 'exam';
kind: string;
status: 'active' | 'completed';
settings: string;
snapshot: string;
answers: string;
flags: string;
current_index: number;
elapsed_sec: number;
question_count: number;
score: number | null;
max_score: number | null;
started_at: number;
updated_at: number;
completed_at: number | null;
}
export const attemptCreateSchema = z.object({
mode: z.enum(['practice', 'exam']).default('practice'),
source: z.discriminatedUnion('type', [
z.object({ type: z.literal('quiz'), quizId: z.string() }),
/** Questions missed in one finished attempt. */
z.object({ type: z.literal('retake'), attemptId: z.string() }),
/** Questions answered wrong most often, across a class or everything. */
z.object({ type: z.literal('missed'), classId: z.string().optional(), quizId: z.string().optional() }),
/** Every question on the given topics within a class. */
z.object({ type: z.literal('topics'), classId: z.string(), topics: z.array(z.string()).min(1).max(50) }),
z.object({ type: z.literal('questions'), questionIds: z.array(z.string()).min(1).max(1000), title: z.string().max(200).optional() }),
]),
shuffleQuestions: z.boolean().default(true),
shuffleOptions: z.boolean().default(true),
count: z.number().int().min(1).max(1000).optional(),
timeLimitMin: z.number().min(1).max(600).nullish(),
});
export type AttemptCreate = z.input<typeof attemptCreateSchema>;
const PINNED_OPTION = /^(all|none|both|neither)\b.*\b(above|these|options|answers)\b|^(a|b|c|d)\s*(and|&)\s*(a|b|c|d)\b/i;
/** Shuffles options but keeps "All of the above"-style choices at the end. */
function shuffleOptions(options: Option[], seed: number): Option[] {
const pinned = options.filter((o) => PINNED_OPTION.test(o.text.trim()));
const free = options.filter((o) => !PINNED_OPTION.test(o.text.trim()));
return [...seededShuffle(free, seed), ...pinned];
}
function loadQuestionRows(db: DB, ids: string[]): (QuestionRow & { quiz_title: string })[] {
if (ids.length === 0) return [];
const out: (QuestionRow & { quiz_title: string })[] = [];
for (let i = 0; i < ids.length; i += 400) {
const batch = ids.slice(i, i + 400);
out.push(
...db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.id IN (${batch.map(() => '?').join(',')})`,
...batch,
),
);
}
const order = new Map(ids.map((id, i) => [id, i]));
return out.sort((a, b) => (order.get(a.id) ?? 0) - (order.get(b.id) ?? 0));
}
/** Question ids ordered from most to least troublesome, using recent responses. */
export function missedQuestionIds(db: DB, scope: { classId?: string; quizId?: string }, limit = 200): string[] {
const where = scope.quizId ? 'AND r.quiz_id = ?' : scope.classId ? 'AND r.class_id = ?' : '';
const params = scope.quizId ? [scope.quizId] : scope.classId ? [scope.classId] : [];
const rows = db.all<{ question_id: string }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(CASE WHEN r.correct = 0 THEN 1 ELSE 0 END) AS wrong,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id
WHERE 1 = 1 ${where}
GROUP BY r.question_id
HAVING wrong > 0 AND (last_correct = 0 OR wrong * 2 >= seen)
ORDER BY last_correct ASC, (wrong * 1.0 / seen) DESC, wrong DESC
LIMIT ?`,
...params,
limit,
);
return rows.map((r) => r.question_id);
}
export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataScoring): string {
const opts = attemptCreateSchema.parse(input);
const src = opts.source;
let quizId: string | null = null;
let classId: string | null = null;
let title = '';
let kind = 'full';
let rows: (QuestionRow & { quiz_title: string })[] = [];
let frozen: SnapshotQuestion[] | null = null;
if (src.type === 'quiz') {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = quiz.title;
rows = db
.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quiz.id)
.map((r) => ({ ...r, quiz_title: quiz.title }));
} else if (src.type === 'retake') {
const prev = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', src.attemptId);
if (!prev) throw notFound('Attempt');
const snap = parseJson<{ questions: SnapshotQuestion[] }>(prev.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(prev.answers, {});
const prevSettings = parseJson<AttemptSettings>(prev.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring });
const missed = snap.questions.filter((q) => !isFullyCorrect(scoreQuestion(q, answers[q.id]?.selected ?? [], prevSettings.sataScoring)));
const current = new Map(loadQuestionRows(db, missed.map((q) => q.id)).map((r) => [r.id, r]));
// Prefer the live question (it may have been corrected since); fall back to the frozen copy.
frozen = missed.map((q) => {
const live = current.get(q.id);
if (!live) return q;
const m = mapQuestion(live);
return { id: m.id, quizId: m.quizId, quizTitle: live.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
quizId = prev.quiz_id;
classId = prev.class_id;
title = `${prev.title.replace(/ · missed$/, '')} · missed`;
kind = 'retake';
} else if (src.type === 'missed') {
const ids = missedQuestionIds(db, { classId: src.classId, quizId: src.quizId });
rows = loadQuestionRows(db, ids);
kind = 'missed';
if (src.quizId) {
const quiz = getQuiz(db, src.quizId);
quizId = quiz.id;
classId = quiz.class_id;
title = `${quiz.title} · missed`;
} else if (src.classId) {
classId = src.classId;
title = `${getClass(db, src.classId).name} · missed questions`;
} else {
title = 'Missed questions';
}
} else if (src.type === 'topics') {
const cls = getClass(db, src.classId);
classId = cls.id;
rows = db.all<QuestionRow & { quiz_title: string }>(
`SELECT q.*, z.title AS quiz_title FROM questions q JOIN quizzes z ON z.id = q.quiz_id
WHERE z.class_id = ? AND q.topic IN (${src.topics.map(() => '?').join(',')}) ORDER BY z.sort_order, q.sort_order`,
cls.id,
...src.topics,
);
title = src.topics.length === 1 ? `${src.topics[0]} · practice` : `${cls.name} · ${src.topics.length} topics`;
kind = 'topics';
} else {
rows = loadQuestionRows(db, src.questionIds);
title = src.title ?? 'Custom practice';
kind = 'custom';
const classes = new Set(rows.map((r) => db.get<{ class_id: string }>('SELECT class_id FROM quizzes WHERE id = ?', r.quiz_id)?.class_id));
if (classes.size === 1) classId = [...classes][0] ?? null;
}
let questions: SnapshotQuestion[] =
frozen ??
rows.map((r) => {
const m = mapQuestion(r);
return { id: m.id, quizId: m.quizId, quizTitle: r.quiz_title, type: m.type, stem: m.stem, options: m.options, rationale: m.rationale, topic: m.topic, difficulty: m.difficulty };
});
if (questions.length === 0) {
throw badRequest(src.type === 'quiz' ? 'This quiz has no questions yet' : src.type === 'missed' || src.type === 'retake' ? 'No missed questions to review' : 'No questions matched');
}
const seed = (Math.random() * 2 ** 32) >>> 0;
if (opts.shuffleQuestions) questions = seededShuffle(questions, seed);
if (opts.count && opts.count < questions.length) questions = questions.slice(0, opts.count);
if (opts.shuffleOptions) questions = questions.map((q, i) => ({ ...q, options: shuffleOptions(q.options, seed + i + 1) }));
const settings: AttemptSettings = {
shuffleQuestions: opts.shuffleQuestions,
shuffleOptions: opts.shuffleOptions,
timeLimitSec: opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null,
sataScoring,
};
const id = newId();
const at = now();
db.run(
`INSERT INTO attempts (id, quiz_id, class_id, title, mode, kind, status, settings, snapshot, question_count, started_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)`,
id,
quizId,
classId,
title,
opts.mode,
kind,
JSON.stringify(settings),
JSON.stringify({ questions }),
questions.length,
at,
at,
);
return id;
}
function loadAttempt(db: DB, id: string): AttemptRow {
const row = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', id);
if (!row) throw notFound('Session');
return row;
}
function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: string[], sata: SataScoring, at: number): void {
const score = scoreQuestion(q, selected, sata);
db.run(
`INSERT INTO responses (attempt_id, question_id, quiz_id, class_id, topic, score, correct, answered_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(attempt_id, question_id) DO UPDATE SET score = excluded.score, correct = excluded.correct, answered_at = excluded.answered_at`,
row.id,
q.id,
q.quizId ?? row.quiz_id,
row.class_id,
q.topic,
score,
isFullyCorrect(score) ? 1 : 0,
at,
);
}
export function mapAttempt(row: AttemptRow) {
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const results =
row.status === 'completed'
? Object.fromEntries(
snapshot.questions.map((q) => {
const selected = answers[q.id]?.selected ?? [];
const score = scoreQuestion(q, selected, settings.sataScoring);
return [q.id, { score, correct: isFullyCorrect(score), answered: selected.length > 0 }];
}),
)
: null;
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
settings,
questions: snapshot.questions,
answers,
flags: parseJson<string[]>(row.flags, []),
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
results,
};
}
export type AttemptDto = ReturnType<typeof mapAttempt>;
export const mapAttemptSummary = (row: AttemptRow) => {
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
return {
id: row.id,
quizId: row.quiz_id,
classId: row.class_id,
title: row.title,
mode: row.mode,
kind: row.kind,
status: row.status,
questionCount: row.question_count,
answeredCount: Object.values(answers).filter((a) => a.selected.length > 0).length,
currentIndex: row.current_index,
elapsedSec: row.elapsed_sec,
score: row.score,
maxScore: row.max_score,
startedAt: row.started_at,
updatedAt: row.updated_at,
completedAt: row.completed_at,
};
};
export type AttemptSummary = ReturnType<typeof mapAttemptSummary>;
export function getAttempt(db: DB, id: string): AttemptDto {
return mapAttempt(loadAttempt(db, id));
}
export const attemptPatchSchema = z.object({
answers: z
.record(z.string(), z.object({ selected: z.array(z.string().max(40)).max(12), checked: z.boolean().optional().default(false) }))
.optional(),
flags: z.array(z.string()).max(1000).optional(),
currentIndex: z.number().int().min(0).optional(),
elapsedSec: z.number().int().min(0).max(1_000_000).optional(),
});
export function patchAttempt(db: DB, id: string, input: z.input<typeof attemptPatchSchema>): AttemptDto {
const patch = attemptPatchSchema.parse(input);
const row = loadAttempt(db, id);
if (row.status !== 'active') throw badRequest('This session is already finished');
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
db.tx(() => {
if (patch.answers) {
const byId = new Map(snapshot.questions.map((q) => [q.id, q]));
for (const [qid, a] of Object.entries(patch.answers)) {
const q = byId.get(qid);
if (!q) continue;
const prev = answers[qid];
// A checked practice answer is final: feedback has been shown.
if (prev?.checked && row.mode === 'practice') continue;
const valid = new Set(q.options.map((o) => o.id));
const selected = [...new Set(a.selected.filter((s) => valid.has(s)))];
const checked = row.mode === 'practice' && a.checked && selected.length > 0;
answers[qid] = { selected, checked, at };
if (checked) recordResponse(db, row, q, selected, settings.sataScoring, at);
}
}
db.run(
'UPDATE attempts SET answers = ?, flags = ?, current_index = ?, elapsed_sec = ?, updated_at = ? WHERE id = ?',
JSON.stringify(answers),
patch.flags ? JSON.stringify(patch.flags) : row.flags,
patch.currentIndex !== undefined ? Math.min(patch.currentIndex, Math.max(0, snapshot.questions.length - 1)) : row.current_index,
patch.elapsedSec !== undefined ? Math.max(row.elapsed_sec, patch.elapsedSec) : row.elapsed_sec,
at,
id,
);
});
return getAttempt(db, id);
}
/**
* Finishes a session. Exams score every question (blank = wrong). Practice sessions that are
* ended early are scored on the questions actually answered.
*/
export function completeAttempt(db: DB, id: string, elapsedSec?: number): AttemptDto {
const row = loadAttempt(db, id);
if (row.status === 'completed') return mapAttempt(row);
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
const at = now();
let score = 0;
let max = 0;
db.tx(() => {
for (const q of snapshot.questions) {
const selected = answers[q.id]?.selected ?? [];
const answered = selected.length > 0;
if (row.mode === 'practice' && !answered) continue;
max += 1;
score += scoreQuestion(q, selected, settings.sataScoring);
const already = row.mode === 'practice' && answers[q.id]?.checked;
if (!already) recordResponse(db, row, q, selected, settings.sataScoring, answers[q.id]?.at ?? at);
}
db.run(
`UPDATE attempts SET status = 'completed', score = ?, max_score = ?, completed_at = ?, updated_at = ?, elapsed_sec = ? WHERE id = ?`,
score,
max,
at,
at,
elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec,
id,
);
});
return getAttempt(db, id);
}
export function deleteAttempt(db: DB, id: string): void {
const row = loadAttempt(db, id);
db.run('DELETE FROM attempts WHERE id = ?', row.id);
}
export function listAttempts(db: DB, filter: { status?: 'active' | 'completed'; quizId?: string; classId?: string; limit?: number }): AttemptSummary[] {
const where: string[] = [];
const params: (string | number)[] = [];
if (filter.status) {
where.push('status = ?');
params.push(filter.status);
}
if (filter.quizId) {
where.push('quiz_id = ?');
params.push(filter.quizId);
}
if (filter.classId) {
where.push('class_id = ?');
params.push(filter.classId);
}
const rows = db.all<AttemptRow>(
`SELECT * FROM attempts ${where.length ? `WHERE ${where.join(' AND ')}` : ''} ORDER BY COALESCE(completed_at, updated_at) DESC LIMIT ?`,
...params,
Math.min(filter.limit ?? 50, 500),
);
return rows.map(mapAttemptSummary);
}

518
server/services/content.ts Normal file
View file

@ -0,0 +1,518 @@
/**
* Classes, folders, quizzes, questions, decks, cards and notes.
* Used by the HTTP routes, the importer and the AI tools, so every write path
* shares one set of validation and ordering rules.
*/
import { z } from 'zod';
import {
CLASS_COLORS,
cardInputSchema,
questionInputSchema,
type Card,
type Difficulty,
type DraftCard,
type DraftQuestion,
type Option,
type Question,
type QuestionType,
} from '../../shared/content.ts';
import type { DB } from '../db.ts';
import { notFound } from '../http.ts';
import { newId, now, parseJson } from '../util.ts';
/* ------------------------------ row types ------------------------------ */
export interface ClassRow {
id: string;
name: string;
code: string;
description: string;
color: string;
archived: number;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface FolderRow {
id: string;
class_id: string;
name: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuizRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
description: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface QuestionRow {
id: string;
quiz_id: string;
type: QuestionType;
stem: string;
options: string;
rationale: string;
topic: string;
difficulty: Difficulty;
tags: string;
sort_order: number;
created_at: number;
updated_at: number;
}
export interface DeckRow extends QuizRow {}
export interface CardRow {
id: string;
deck_id: string;
front: string;
back: string;
hint: string;
tags: string;
sort_order: number;
suspended: number;
state: number;
due: number | null;
stability: number;
difficulty: number;
scheduled_days: number;
learning_steps: number;
reps: number;
lapses: number;
last_review: number | null;
introduced_at: number | null;
created_at: number;
updated_at: number;
}
export interface NoteRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
content: string;
kind: string;
source: string;
sort_order: number;
created_at: number;
updated_at: number;
}
/* ------------------------------ mappers ------------------------------ */
export const mapClass = (r: ClassRow) => ({
id: r.id,
name: r.name,
code: r.code,
description: r.description,
color: r.color,
archived: Boolean(r.archived),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type ClassDto = ReturnType<typeof mapClass>;
export const mapFolder = (r: FolderRow) => ({ id: r.id, classId: r.class_id, name: r.name, sortOrder: r.sort_order });
export const mapQuiz = (r: QuizRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
description: r.description,
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export const mapDeck = mapQuiz;
export function mapQuestion(r: QuestionRow): Question {
return {
id: r.id,
quizId: r.quiz_id,
type: r.type,
stem: r.stem,
options: parseJson<Option[]>(r.options, []).map((o) => ({ id: o.id, text: o.text, correct: Boolean(o.correct), why: o.why ?? '' })),
rationale: r.rationale,
topic: r.topic,
difficulty: r.difficulty,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export function mapCard(r: CardRow): Card {
return {
id: r.id,
deckId: r.deck_id,
front: r.front,
back: r.back,
hint: r.hint,
tags: parseJson<string[]>(r.tags, []),
sortOrder: r.sort_order,
suspended: Boolean(r.suspended),
state: r.state,
due: r.due,
stability: r.stability,
difficulty: r.difficulty,
scheduledDays: r.scheduled_days,
reps: r.reps,
lapses: r.lapses,
lastReview: r.last_review,
createdAt: r.created_at,
updatedAt: r.updated_at,
};
}
export const mapNote = (r: NoteRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
content: r.content,
kind: r.kind,
source: parseJson<Record<string, unknown>>(r.source, {}),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
/* ------------------------------ lookups ------------------------------ */
export function getClass(db: DB, id: string): ClassRow {
const row = db.get<ClassRow>('SELECT * FROM classes WHERE id = ?', id);
if (!row) throw notFound('Class');
return row;
}
export function getFolder(db: DB, id: string): FolderRow {
const row = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', id);
if (!row) throw notFound('Folder');
return row;
}
export function getQuiz(db: DB, id: string): QuizRow {
const row = db.get<QuizRow>('SELECT * FROM quizzes WHERE id = ?', id);
if (!row) throw notFound('Quiz');
return row;
}
export function getDeck(db: DB, id: string): DeckRow {
const row = db.get<DeckRow>('SELECT * FROM decks WHERE id = ?', id);
if (!row) throw notFound('Deck');
return row;
}
export function getNote(db: DB, id: string): NoteRow {
const row = db.get<NoteRow>('SELECT * FROM notes WHERE id = ?', id);
if (!row) throw notFound('Note');
return row;
}
export function listQuestions(db: DB, quizId: string): Question[] {
return db.all<QuestionRow>('SELECT * FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId).map(mapQuestion);
}
export function listCards(db: DB, deckId: string): Card[] {
return db.all<CardRow>('SELECT * FROM cards WHERE deck_id = ? ORDER BY sort_order, created_at', deckId).map(mapCard);
}
/** Resolves and validates a folder id against a class; returns null for "no folder". */
export function resolveFolder(db: DB, classId: string, folderId: string | null | undefined): string | null {
if (!folderId) return null;
const folder = db.get<FolderRow>('SELECT * FROM folders WHERE id = ?', folderId);
return folder && folder.class_id === classId ? folder.id : null;
}
function nextSort(db: DB, table: string, where: string, value: string): number {
const row = db.get<{ m: number | null }>(`SELECT MAX(sort_order) AS m FROM ${table} WHERE ${where} = ?`, value);
return (row?.m ?? -1) + 1;
}
const touch = (db: DB, table: 'quizzes' | 'decks' | 'classes', id: string) => db.run(`UPDATE ${table} SET updated_at = ? WHERE id = ?`, now(), id);
/* ------------------------------ classes + folders ------------------------------ */
export const classInputSchema = z.object({
name: z.string().trim().min(1, 'Name is required').max(120),
code: z.string().trim().max(40).optional().default(''),
description: z.string().max(2000).optional().default(''),
color: z.enum(CLASS_COLORS).optional(),
});
export function createClass(db: DB, input: z.input<typeof classInputSchema>): ClassRow {
const data = classInputSchema.parse(input);
const id = newId();
const at = now();
const count = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM classes')?.n ?? 0;
const color = data.color ?? CLASS_COLORS[count % CLASS_COLORS.length];
db.run(
'INSERT INTO classes (id, name, code, description, color, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
data.name,
data.code,
data.description,
color,
count,
at,
at,
);
return getClass(db, id);
}
export function createFolder(db: DB, classId: string, name: string): FolderRow {
getClass(db, classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO folders (id, class_id, name, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)',
id,
classId,
name.trim().slice(0, 120) || 'Untitled folder',
nextSort(db, 'folders', 'class_id', classId),
at,
at,
);
return getFolder(db, id);
}
/* ------------------------------ quizzes + questions ------------------------------ */
type ParsedQuestion = z.output<typeof questionInputSchema>;
function optionsJson(q: ParsedQuestion, keep?: Option[]): string {
const used = new Set<string>();
const options: Option[] = q.options.map((o) => {
let id = o.id && !used.has(o.id) ? o.id : undefined;
// Keep option ids stable across edits when the text is unchanged, so old attempts still line up.
if (!id && keep) id = keep.find((k) => k.text === o.text && !used.has(k.id))?.id;
if (!id) id = newId().slice(0, 8);
used.add(id);
return { id, text: o.text, correct: o.correct, why: o.why ?? '' };
});
return JSON.stringify(options);
}
export function insertQuestions(db: DB, quizId: string, inputs: unknown[], atIndex?: number): Question[] {
const parsed = inputs.map((q) => questionInputSchema.parse(q));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'questions', 'quiz_id', quizId);
if (atIndex !== undefined) {
const ordered = db.all<{ id: string }>('SELECT id FROM questions WHERE quiz_id = ? ORDER BY sort_order, created_at', quizId);
const insertAt = Math.max(0, Math.min(atIndex, ordered.length));
ordered.forEach((row, i) => db.run('UPDATE questions SET sort_order = ? WHERE id = ?', i < insertAt ? i : i + parsed.length, row.id));
sort = insertAt;
}
for (const q of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO questions (id, quiz_id, type, stem, options, rationale, topic, difficulty, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
quizId,
q.type,
q.stem,
optionsJson(q),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'quizzes', quizId);
});
const all = listQuestions(db, quizId);
return ids.map((id) => all.find((q) => q.id === id)!);
}
export function updateQuestion(db: DB, id: string, input: unknown): Question {
const existing = db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id);
if (!existing) throw notFound('Question');
const q = questionInputSchema.parse(input);
db.run(
'UPDATE questions SET type = ?, stem = ?, options = ?, rationale = ?, topic = ?, difficulty = ?, tags = ?, updated_at = ? WHERE id = ?',
q.type,
q.stem,
optionsJson(q, mapQuestion(existing).options),
q.rationale,
q.topic,
q.difficulty,
JSON.stringify(q.tags),
now(),
id,
);
touch(db, 'quizzes', existing.quiz_id);
return mapQuestion(db.get<QuestionRow>('SELECT * FROM questions WHERE id = ?', id)!);
}
export function createQuiz(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; questions?: unknown[] },
): QuizRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO quizzes (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'quizzes', 'class_id', input.classId),
at,
at,
);
if (input.questions?.length) insertQuestions(db, id, input.questions);
});
return getQuiz(db, id);
}
export function duplicateQuiz(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): QuizRow {
const quiz = getQuiz(db, id);
const questions = listQuestions(db, id);
const classId = target?.classId ?? quiz.class_id;
return createQuiz(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === quiz.class_id ? quiz.folder_id : null,
title: target?.title ?? `${quiz.title} (copy)`,
description: quiz.description,
questions: questions.map((q) => ({
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why })),
rationale: q.rationale,
topic: q.topic,
difficulty: q.difficulty,
tags: q.tags,
})),
});
}
/* ------------------------------ decks + cards ------------------------------ */
export function insertCards(db: DB, deckId: string, inputs: unknown[]): Card[] {
const parsed = inputs.map((c) => cardInputSchema.parse(c));
const at = now();
const ids: string[] = [];
db.tx(() => {
let sort = nextSort(db, 'cards', 'deck_id', deckId);
for (const c of parsed) {
const id = newId();
ids.push(id);
db.run(
'INSERT INTO cards (id, deck_id, front, back, hint, tags, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
deckId,
c.front,
c.back,
c.hint,
JSON.stringify(c.tags),
sort,
at,
at,
);
sort += 1;
}
touch(db, 'decks', deckId);
});
const all = listCards(db, deckId);
return ids.map((id) => all.find((c) => c.id === id)!);
}
export function updateCard(db: DB, id: string, input: unknown): Card {
const existing = db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id);
if (!existing) throw notFound('Card');
const c = cardInputSchema.parse(input);
db.run('UPDATE cards SET front = ?, back = ?, hint = ?, tags = ?, updated_at = ? WHERE id = ?', c.front, c.back, c.hint, JSON.stringify(c.tags), now(), id);
touch(db, 'decks', existing.deck_id);
return mapCard(db.get<CardRow>('SELECT * FROM cards WHERE id = ?', id)!);
}
export function createDeck(
db: DB,
input: { classId: string; folderId?: string | null; title: string; description?: string; cards?: unknown[] },
): DeckRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.tx(() => {
db.run(
'INSERT INTO decks (id, class_id, folder_id, title, description, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim(),
input.description ?? '',
nextSort(db, 'decks', 'class_id', input.classId),
at,
at,
);
if (input.cards?.length) insertCards(db, id, input.cards);
});
return getDeck(db, id);
}
export function duplicateDeck(db: DB, id: string, target?: { classId?: string; folderId?: string | null; title?: string }): DeckRow {
const deck = getDeck(db, id);
const cards = listCards(db, id);
const classId = target?.classId ?? deck.class_id;
return createDeck(db, {
classId,
folderId: target?.folderId !== undefined ? target.folderId : classId === deck.class_id ? deck.folder_id : null,
title: target?.title ?? `${deck.title} (copy)`,
description: deck.description,
cards: cards.map((c) => ({ front: c.front, back: c.back, hint: c.hint, tags: c.tags })),
});
}
/* ------------------------------ notes ------------------------------ */
export function createNote(
db: DB,
input: { classId: string; folderId?: string | null; title: string; content?: string; kind?: string; source?: unknown },
): NoteRow {
getClass(db, input.classId);
const id = newId();
const at = now();
db.run(
'INSERT INTO notes (id, class_id, folder_id, title, content, kind, source, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
input.title.trim().slice(0, 200) || 'Untitled note',
input.content ?? '',
input.kind ?? 'note',
JSON.stringify(input.source ?? {}),
nextSort(db, 'notes', 'class_id', input.classId),
at,
at,
);
return getNote(db, id);
}
/* ------------------------------ draft conversion ------------------------------ */
/** Turns a (validated) draft question into a strict question input, folding the citation into the rationale. */
export function draftQuestionToInput(q: DraftQuestion) {
const source = q.source?.trim();
const rationale = source && !q.rationale.includes(source) ? `${q.rationale.trim()}\n\n*Source: ${source}*`.trim() : q.rationale;
return {
type: q.type,
stem: q.stem,
options: q.options.map((o) => ({ text: o.text, correct: o.correct, why: o.why ?? '' })),
rationale,
topic: (q.topic ?? '').slice(0, 80),
difficulty: q.difficulty ?? 'medium',
tags: (q.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12),
};
}
export function draftCardToInput(c: DraftCard) {
return { front: c.front, back: c.back, hint: c.hint ?? '', tags: (c.tags ?? []).map((t) => t.slice(0, 40)).filter(Boolean).slice(0, 12) };
}

View file

@ -0,0 +1,500 @@
/**
* Uploaded study materials: storage, the background processing queue
* (extract -> optional AI transcription -> chunk -> index) and retrieval.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { resolveModel, runChat } from '../ai/service.ts';
import { AIError, type ContentPart } from '../ai/types.ts';
import type { DB } from '../db.ts';
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
import { detectKind, extractDocument } from '../documents/extract.ts';
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { ftsQuery, newId, now, parseJson } from '../util.ts';
import { getClass, resolveFolder } from './content.ts';
export interface DocumentRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
filename: string;
mime: string;
kind: DocKind;
size: number;
sha256: string;
storage_name: string;
status: 'pending' | 'processing' | 'ready' | 'failed';
error: string;
progress: number;
page_count: number | null;
char_count: number;
token_count: number;
chunk_count: number;
needs_ocr: number;
warnings: string;
created_at: number;
updated_at: number;
}
export const mapDocument = (r: DocumentRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
filename: r.filename,
mime: r.mime,
docKind: r.kind,
size: r.size,
status: r.status,
error: r.error,
progress: r.progress,
pageCount: r.page_count,
charCount: r.char_count,
tokenCount: r.token_count,
chunkCount: r.chunk_count,
needsOcr: Boolean(r.needs_ocr),
warnings: parseJson<string[]>(r.warnings, []),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type DocumentDto = ReturnType<typeof mapDocument>;
const MIME_BY_KIND: Record<DocKind, string> = {
pdf: 'application/pdf',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
text: 'text/plain; charset=utf-8',
markdown: 'text/markdown; charset=utf-8',
image: 'application/octet-stream',
};
function imageMime(bytes: Uint8Array): string {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png';
if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif';
if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp';
return 'application/octet-stream';
}
export function getDocument(db: DB, id: string): DocumentRow {
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) throw notFound('Document');
return row;
}
export function documentPath(app: AppContext, row: Pick<DocumentRow, 'storage_name'>): string {
// basename() guarantees a stored name can never escape the uploads folder.
return path.join(app.config.uploadsDir, path.basename(row.storage_name));
}
function titleFromFilename(filename: string): string {
const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim();
return (base || 'Untitled document').slice(0, 200);
}
/* ------------------------------ ingest ------------------------------ */
export function ingestUpload(
app: AppContext,
input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string },
): DocumentRow {
const { db } = app;
getClass(db, input.classId);
if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`);
if (input.bytes.length > app.config.maxUploadBytes) {
throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
}
const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload';
const kind = detectKind(filename, input.mime, input.bytes);
if (!kind) {
throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`);
}
const id = newId();
const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind];
const storageName = `${id}${ext}`;
fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes);
const at = now();
const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1;
db.run(
`INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`,
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
(input.title?.trim() || titleFromFilename(filename)).slice(0, 200),
filename,
kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind],
kind,
input.bytes.length,
crypto.createHash('sha256').update(input.bytes).digest('hex'),
storageName,
sort,
at,
at,
);
enqueue(app, id);
return getDocument(db, id);
}
export function deleteDocument(app: AppContext, id: string): void {
const row = getDocument(app.db, id);
app.db.tx(() => {
app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id);
app.db.run('DELETE FROM documents WHERE id = ?', id);
});
fs.rmSync(documentPath(app, row), { force: true });
}
/* ------------------------------ processing queue ------------------------------ */
const queues = new WeakMap<DB, { pending: string[]; running: boolean; ocr: Set<string>; idle: Promise<void>; wake: () => void }>();
function queueFor(app: AppContext) {
let q = queues.get(app.db);
if (!q) {
q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined };
queues.set(app.db, q);
}
return q;
}
export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void {
const q = queueFor(app);
if (opts.ocr) q.ocr.add(id);
if (!q.pending.includes(id)) q.pending.push(id);
if (!q.running) {
q.running = true;
q.idle = (async () => {
// Yield first so the HTTP response for the upload is sent before heavy parsing starts.
await new Promise((r) => setTimeout(r, 10));
while (q.pending.length > 0) {
const next = q.pending.shift()!;
const forceOcr = q.ocr.delete(next);
try {
await processDocument(app, next, { forceOcr });
} catch (e) {
app.log('error', `Document ${next} failed unexpectedly`, e);
}
}
q.running = false;
})();
}
}
/** Resolves when the queue is empty. Used by tests and graceful shutdown. */
export function queueIdle(app: AppContext): Promise<void> {
return queueFor(app).idle;
}
/** Re-queues anything a restart interrupted. */
export function resumeQueue(app: AppContext): number {
const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`);
for (const r of rows) enqueue(app, r.id);
return rows.length;
}
const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
- Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information.
- Keep headings, lists and tables (as Markdown tables).
- For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown.
- For a multi-page document, start each page with a line of the form "## Page N".
- If something is illegible, write [illegible].
Return only the transcription.`;
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
const data = Buffer.from(bytes).toString('base64');
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
}
let result;
try {
const vision = resolveModel(app, 'vision');
if (!vision.model.capabilities.vision) return null;
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
} catch (err) {
if (err instanceof AIError && err.code === 'not_configured') return null;
throw err;
}
const text = result.text.trim();
if (!text) return [];
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
if (body) sections.push({ text: body, page: Number(pages[i]) });
}
return sections;
}
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
const { db } = app;
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) return;
const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id);
db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id);
try {
const file = documentPath(app, row);
if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.');
const bytes = new Uint8Array(await fs.promises.readFile(file));
let lastWrite = 0;
const result = await extractDocument({
bytes,
filename: row.filename,
mime: row.mime,
onProgress: (done, total) => {
const t = Date.now();
if (t - lastWrite > 400 && total > 0) {
lastWrite = t;
setProgress((done / total) * 0.8);
}
},
});
const warnings = [...result.warnings];
let sections = result.sections;
let needsOcr = result.needsOcr;
if (needsOcr || opts.forceOcr) {
setProgress(0.82);
if (row.kind === 'image' || row.kind === 'pdf') {
try {
const transcribed = await transcribeWithVision(app, row, bytes);
if (transcribed === null) {
warnings.push(
row.kind === 'image'
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
);
} else if (transcribed.length > 0) {
sections = transcribed;
needsOcr = false;
} else {
warnings.push('The AI model did not find any readable text in this file.');
}
} catch (err) {
warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
}
const chunks = chunkSections(sections);
const chars = sections.reduce((n, s) => n + s.text.length, 0);
const tokens = chunks.reduce((n, c) => n + c.tokens, 0);
db.tx(() => {
db.run('DELETE FROM chunks WHERE document_id = ?', id);
const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id);
if (!current) return; // deleted while processing
for (const c of chunks) {
db.run(
'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
current.class_id,
c.seq,
c.text,
c.heading,
c.pageStart,
c.pageEnd,
c.tokens,
);
}
db.run(
`UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`,
result.pageCount ?? null,
chars,
tokens,
chunks.length,
needsOcr ? 1 : 0,
JSON.stringify([...new Set(warnings)].slice(0, 8)),
now(),
id,
);
});
} catch (err) {
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id);
}
}
/* ------------------------------ retrieval ------------------------------ */
export interface ChunkHit {
id: number;
documentId: string;
documentTitle: string;
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
score?: number;
}
type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number };
const mapHit = (r: ChunkJoin): ChunkHit => ({
id: r.id,
documentId: r.document_id,
documentTitle: r.doc_title,
seq: r.seq,
text: r.text,
heading: r.heading,
pageStart: r.page_start,
pageEnd: r.page_end,
tokens: r.tokens,
score: r.score,
});
export interface ChunkScope {
classId?: string | null;
documentIds?: string[];
}
function scopeSql(scope: ChunkScope): { sql: string; params: string[] } {
if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds };
if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] };
return { sql: '', params: [] };
}
/** BM25 full-text search over document chunks. */
export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] {
const match = ftsQuery(query, 'or');
if (!match) return [];
const { sql, params } = scopeSql(scope);
try {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id
WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`,
match,
...params,
limit,
)
.map(mapHit);
} catch {
return [];
}
}
export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`,
documentId,
offset,
limit,
)
.map(mapHit);
}
export function chunkById(db: DB, id: number): ChunkHit | null {
const row = db.get<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`,
id,
);
return row ? mapHit(row) : null;
}
export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' | 'pageEnd'>): string {
if (hit.pageStart === null) return hit.documentTitle;
return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`;
}
/**
* Chooses source material for a generation job and splits it into `groups` batches that
* each fit `budgetTokens`.
* - With a focus, the best-matching chunks are used (kept in document order).
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
* so every part of the document is represented.
*/
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
const { sql, params } = scopeSql(scope);
if (!sql) return [];
const all = db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`,
...params,
)
.map(mapHit);
if (all.length === 0) return [];
const capacity = opts.budgetTokens * opts.groups;
const total = all.reduce((n, c) => n + c.tokens, 0);
let chosen = all;
if (opts.focus?.trim()) {
const ranked = searchChunks(db, opts.focus, scope, 400);
if (ranked.length >= 3) {
const keep = new Set<number>();
let used = 0;
for (const hit of ranked) {
if (used + hit.tokens > capacity) break;
keep.add(hit.id);
used += hit.tokens;
}
chosen = all.filter((c) => keep.has(c.id));
}
}
if (chosen === all && total > capacity) {
// Even sampling: walk the document and keep chunks at a steady stride.
const stride = total / capacity;
const picked: ChunkHit[] = [];
let acc = 0;
let next = 0;
for (const c of all) {
if (acc >= next) {
picked.push(c);
next += c.tokens * stride;
}
acc += c.tokens;
}
chosen = picked;
}
// Contiguous groups of roughly equal size, none above the budget.
const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0);
const groupCount = Math.max(1, Math.min(opts.groups, chosen.length));
const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount));
const groups: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const c of chosen) {
if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) {
groups.push(current);
current = [];
size = 0;
}
if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full
current.push(c);
size += c.tokens;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function formatExcerpts(chunks: ChunkHit[]): string {
return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n');
}
export { estimateTokens };

309
server/services/srs.ts Normal file
View file

@ -0,0 +1,309 @@
/**
* Spaced repetition with FSRS (the algorithm Anki adopted in 2023), via ts-fsrs.
* Scheduling state lives on each card row; every review is logged so it can be undone
* and analysed later.
*/
import { createEmptyCard, fsrs, Rating, State, type Card as FsrsCard, type Grade } from 'ts-fsrs';
import type { DB } from '../db.ts';
import { badRequest, notFound } from '../http.ts';
import type { AppSettings } from '../settings.ts';
import { DAY_MS, now as clock, parseJson, startOfDay } from '../util.ts';
import { mapCard, type CardRow } from './content.ts';
export const LEARN_AHEAD_MS = 20 * 60_000;
export type RatingValue = 1 | 2 | 3 | 4;
export const RATING_LABELS: Record<RatingValue, string> = { 1: 'Again', 2: 'Hard', 3: 'Good', 4: 'Easy' };
const schedulers = new Map<string, ReturnType<typeof fsrs>>();
function scheduler(retention: number, fuzz: boolean) {
const key = `${retention}:${fuzz}`;
let s = schedulers.get(key);
if (!s) {
s = fsrs({ request_retention: retention, enable_fuzz: fuzz, enable_short_term: true, maximum_interval: 36500 });
schedulers.set(key, s);
}
return s;
}
function toFsrs(row: CardRow, at: number): FsrsCard {
if (row.state === State.New || row.due === null) return createEmptyCard(new Date(at));
return {
due: new Date(row.due),
stability: row.stability,
difficulty: row.difficulty,
scheduled_days: row.scheduled_days,
learning_steps: row.learning_steps,
reps: row.reps,
lapses: row.lapses,
state: row.state as State,
last_review: row.last_review ? new Date(row.last_review) : undefined,
} as FsrsCard;
}
export function intervalLabel(ms: number): string {
if (ms < 60_000) return '<1m';
const minutes = ms / 60_000;
if (minutes < 60) return `${Math.round(minutes)}m`;
const hours = minutes / 60;
if (hours < 24) return `${Math.round(hours)}h`;
const days = hours / 24;
if (days < 30.5) return `${Math.round(days)}d`;
const months = days / 30.44;
if (months < 12) return `${months.toFixed(months < 3 ? 1 : 0).replace(/\.0$/, '')}mo`;
return `${(days / 365.25).toFixed(1).replace(/\.0$/, '')}y`;
}
export interface RatingPreview {
rating: RatingValue;
label: string;
interval: string;
due: number;
}
export function previewRatings(row: CardRow, settings: AppSettings, at: number): RatingPreview[] {
// Previews never use fuzz so the labels are stable while the card is on screen.
const options = scheduler(settings.desiredRetention, false).repeat(toFsrs(row, at), new Date(at));
return ([1, 2, 3, 4] as RatingValue[]).map((rating) => {
const due = options[rating as Grade].card.due.getTime();
return { rating, label: RATING_LABELS[rating], interval: intervalLabel(due - at), due };
});
}
export interface QueueScope {
deckId?: string;
classId?: string;
}
function scopeSql(scope: QueueScope): { where: string; params: string[] } {
if (scope.deckId) return { where: 'AND c.deck_id = ?', params: [scope.deckId] };
if (scope.classId) return { where: 'AND d.class_id = ?', params: [scope.classId] };
return { where: '', params: [] };
}
const BASE = `FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.suspended = 0 AND k.archived = 0`;
export interface DueCounts {
learning: number;
review: number;
new: number;
total: number;
}
/** New cards still allowed today under the global daily limit. */
export function newCardsLeftToday(db: DB, settings: AppSettings, at: number): number {
const since = startOfDay(at, settings.timezone);
const introduced = db.get<{ n: number }>('SELECT COUNT(*) AS n FROM cards WHERE introduced_at >= ?', since)?.n ?? 0;
return Math.max(0, settings.newCardsPerDay - introduced);
}
export function dueCounts(db: DB, scope: QueueScope, settings: AppSettings, at = clock()): DueCounts {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const row = db.get<{ learning: number; review: number; fresh: number }>(
`SELECT
COALESCE(SUM(CASE WHEN c.state IN (1,3) AND c.due <= ? THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN c.state = 2 AND c.due < ? THEN 1 ELSE 0 END), 0) AS review,
COALESCE(SUM(CASE WHEN c.state = 0 THEN 1 ELSE 0 END), 0) AS fresh
${BASE} ${where}`,
at + LEARN_AHEAD_MS,
endOfDay,
...params,
)!;
const fresh = Math.min(row.fresh, newCardsLeftToday(db, settings, at));
return { learning: row.learning, review: row.review, new: fresh, total: row.learning + row.review + fresh };
}
export interface QueueCard {
card: ReturnType<typeof mapCard>;
deckTitle: string;
classId: string;
className: string;
classColor: string;
previews: RatingPreview[];
}
type JoinedRow = CardRow & { deck_title: string; class_id: string; class_name: string; class_color: string };
const SELECT = `SELECT c.*, d.title AS deck_title, d.class_id AS class_id, k.name AS class_name, k.color AS class_color`;
function toQueueCard(row: JoinedRow, settings: AppSettings, at: number): QueueCard {
return {
card: mapCard(row),
deckTitle: row.deck_title,
classId: row.class_id,
className: row.class_name,
classColor: row.class_color,
previews: previewRatings(row, settings, at),
};
}
/**
* Builds the study queue: learning cards first (they are time-critical), then reviews
* due today with new cards spread evenly among them.
*/
export function buildQueue(db: DB, scope: QueueScope, settings: AppSettings, limit = 100, at = clock()): { cards: QueueCard[]; counts: DueCounts } {
const { where, params } = scopeSql(scope);
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const learning = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state IN (1,3) AND c.due <= ? ORDER BY c.due LIMIT ?`, ...params, at + LEARN_AHEAD_MS, limit);
const reviews = db.all<JoinedRow>(`${SELECT} ${BASE} ${where} AND c.state = 2 AND c.due < ? ORDER BY c.due LIMIT ?`, ...params, endOfDay, limit);
const newLeft = newCardsLeftToday(db, settings, at);
const fresh =
newLeft > 0
? db.all<JoinedRow>(
`${SELECT} ${BASE} ${where} AND c.state = 0 ORDER BY k.sort_order, d.sort_order, d.created_at, c.sort_order, c.created_at LIMIT ?`,
...params,
Math.min(newLeft, limit),
)
: [];
const mixed: JoinedRow[] = [];
const every = fresh.length > 0 ? Math.max(1, Math.floor(reviews.length / fresh.length)) : 0;
let f = 0;
reviews.forEach((r, i) => {
mixed.push(r);
if (every > 0 && (i + 1) % every === 0 && f < fresh.length) mixed.push(fresh[f++]);
});
while (f < fresh.length) mixed.push(fresh[f++]);
const cards = [...learning, ...mixed].slice(0, limit).map((r) => toQueueCard(r, settings, at));
return { cards, counts: dueCounts(db, scope, settings, at) };
}
const SCHEDULE_FIELDS = ['state', 'due', 'stability', 'difficulty', 'scheduled_days', 'learning_steps', 'reps', 'lapses', 'last_review', 'introduced_at'] as const;
export function answerCard(
db: DB,
cardId: string,
rating: RatingValue,
settings: AppSettings,
opts: { at?: number; fuzz?: boolean } = {},
): QueueCard {
const at = opts.at ?? clock();
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
if (![1, 2, 3, 4].includes(rating)) throw badRequest('Rating must be 1-4');
const result = scheduler(settings.desiredRetention, opts.fuzz ?? true).next(toFsrs(row, at), new Date(at), rating as Grade);
const next = result.card;
const prev: Record<string, unknown> = {};
for (const f of SCHEDULE_FIELDS) prev[f] = row[f];
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?,
introduced_at = COALESCE(introduced_at, ?) WHERE id = ?`,
next.state,
next.due.getTime(),
next.stability,
next.difficulty,
next.scheduled_days,
next.learning_steps,
next.reps,
next.lapses,
at,
at,
cardId,
);
db.run(
'INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
cardId,
row.deck_id,
row.class_id,
rating,
row.state,
next.scheduled_days,
'srs',
JSON.stringify(prev),
at,
);
});
const updated = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId)!;
return toQueueCard(updated, settings, at);
}
/** Reverts the most recent scheduled review of a card and returns it to its previous state. */
export function undoReview(db: DB, cardId: string, settings: AppSettings, at = clock()): QueueCard {
const review = db.get<{ id: number; prev: string }>(`SELECT id, prev FROM reviews WHERE card_id = ? AND mode = 'srs' ORDER BY id DESC LIMIT 1`, cardId);
if (!review || !review.prev) throw badRequest('Nothing to undo for this card');
const prev = parseJson<Record<string, number | null>>(review.prev, {});
db.tx(() => {
db.run(
`UPDATE cards SET state = ?, due = ?, stability = ?, difficulty = ?, scheduled_days = ?, learning_steps = ?, reps = ?, lapses = ?, last_review = ?, introduced_at = ? WHERE id = ?`,
prev.state ?? 0,
prev.due ?? null,
prev.stability ?? 0,
prev.difficulty ?? 0,
prev.scheduled_days ?? 0,
prev.learning_steps ?? 0,
prev.reps ?? 0,
prev.lapses ?? 0,
prev.last_review ?? null,
prev.introduced_at ?? null,
cardId,
);
db.run('DELETE FROM reviews WHERE id = ?', review.id);
});
const row = db.get<JoinedRow>(`${SELECT} FROM cards c JOIN decks d ON d.id = c.deck_id JOIN classes k ON k.id = d.class_id WHERE c.id = ?`, cardId);
if (!row) throw notFound('Card');
return toQueueCard(row, settings, at);
}
export function resetCards(db: DB, where: { cardId?: string; deckId?: string }): number {
const sql = `UPDATE cards SET state = 0, due = NULL, stability = 0, difficulty = 0, scheduled_days = 0, learning_steps = 0, reps = 0, lapses = 0, last_review = NULL, introduced_at = NULL WHERE `;
if (where.cardId) return db.run(`${sql} id = ?`, where.cardId).changes;
if (where.deckId) return db.run(`${sql} deck_id = ?`, where.deckId).changes;
return 0;
}
/** Logs a self-graded cram flip so it counts toward activity without touching the schedule. */
export function logCram(db: DB, cardId: string, known: boolean, at = clock()): void {
const row = db.get<{ deck_id: string; class_id: string; state: number }>(
'SELECT c.deck_id, d.class_id, c.state FROM cards c JOIN decks d ON d.id = c.deck_id WHERE c.id = ?',
cardId,
);
if (!row) return;
db.run(
`INSERT INTO reviews (card_id, deck_id, class_id, rating, state_before, scheduled_days, mode, prev, reviewed_at) VALUES (?, ?, ?, ?, ?, 0, 'cram', '', ?)`,
cardId,
row.deck_id,
row.class_id,
known ? Rating.Good : Rating.Again,
row.state,
at,
);
}
export interface DeckStats {
total: number;
new: number;
learning: number;
young: number;
mature: number;
suspended: number;
dueNow: number;
}
export function deckStats(db: DB, deckId: string, settings: AppSettings, at = clock()): DeckStats {
const endOfDay = startOfDay(at, settings.timezone) + DAY_MS;
const r = db.get<Record<string, number>>(
`SELECT COUNT(*) AS total,
COALESCE(SUM(suspended), 0) AS suspended,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 0 THEN 1 ELSE 0 END), 0) AS fresh,
COALESCE(SUM(CASE WHEN suspended = 0 AND state IN (1,3) THEN 1 ELSE 0 END), 0) AS learning,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days < 21 THEN 1 ELSE 0 END), 0) AS young,
COALESCE(SUM(CASE WHEN suspended = 0 AND state = 2 AND scheduled_days >= 21 THEN 1 ELSE 0 END), 0) AS mature,
COALESCE(SUM(CASE WHEN suspended = 0 AND ((state IN (1,3) AND due <= ?) OR (state = 2 AND due < ?)) THEN 1 ELSE 0 END), 0) AS due_now
FROM cards WHERE deck_id = ?`,
at + LEARN_AHEAD_MS,
endOfDay,
deckId,
)!;
return { total: r.total, new: r.fresh, learning: r.learning, young: r.young, mature: r.mature, suspended: r.suspended, dueNow: r.due_now };
}
/** Reviews coming due per day for the next `days` days (index 0 = today, including overdue). */
export function forecast(db: DB, scope: QueueScope, settings: AppSettings, days = 14, at = clock()): number[] {
const { where, params } = scopeSql(scope);
const start = startOfDay(at, settings.timezone);
const rows = db.all<{ due: number }>(`SELECT c.due AS due ${BASE} ${where} AND c.state != 0 AND c.due < ?`, ...params, start + days * DAY_MS);
const out = new Array<number>(days).fill(0);
for (const r of rows) out[Math.max(0, Math.min(days - 1, Math.floor((r.due - start) / DAY_MS)))] += 1;
return out;
}

185
server/services/stats.ts Normal file
View file

@ -0,0 +1,185 @@
/** Study activity, streaks, accuracy and weak-topic analytics. All derived from the review and response logs. */
import type { DB } from '../db.ts';
import type { AppSettings } from '../settings.ts';
import { addDaysToKey, DAY_MS, dayKey, now as clock, startOfDay } from '../util.ts';
export interface DayActivity {
date: string;
cards: number;
questions: number;
total: number;
}
/** Per-day totals for the trailing `days` days (oldest first, always dense). */
export function activityByDay(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): DayActivity[] {
const tz = settings.timezone;
const today = dayKey(at, tz);
const since = startOfDay(at, tz) - (days - 1) * DAY_MS - DAY_MS; // one extra day absorbs DST shifts
const map = new Map<string, DayActivity>();
const bump = (ts: number, field: 'cards' | 'questions') => {
const key = dayKey(ts, tz);
let d = map.get(key);
if (!d) {
d = { date: key, cards: 0, questions: 0, total: 0 };
map.set(key, d);
}
d[field] += 1;
d.total += 1;
};
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
for (const r of db.all<{ t: number }>(`SELECT reviewed_at AS t FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)) bump(r.t, 'cards');
for (const r of db.all<{ t: number }>(`SELECT answered_at AS t FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) bump(r.t, 'questions');
const out: DayActivity[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
out.push(map.get(key) ?? { date: key, cards: 0, questions: 0, total: 0 });
}
return out;
}
export interface Streak {
current: number;
longest: number;
/** Today already meets the daily goal. */
todayDone: boolean;
todayTotal: number;
goal: number;
}
export function computeStreak(days: DayActivity[], goal: number): Streak {
const met = days.map((d) => d.total >= goal);
const todayDone = met[met.length - 1] ?? false;
let current = 0;
// Today only counts once the goal is met; an unfinished today does not break yesterday's streak.
for (let i = met.length - (todayDone ? 1 : 2); i >= 0 && met[i]; i -= 1) current += 1;
let longest = 0;
let run = 0;
for (const m of met) {
run = m ? run + 1 : 0;
longest = Math.max(longest, run);
}
return { current, longest, todayDone, todayTotal: days[days.length - 1]?.total ?? 0, goal };
}
export interface TopicStat {
topic: string;
classId: string | null;
className: string;
classColor: string;
seen: number;
accuracy: number;
/** Distinct questions available on this topic right now. */
questions: number;
}
/** Accuracy per topic over the trailing window. Topics need a few answers before they count. */
export function topicStats(db: DB, scope: { classId?: string }, windowDays = 90, at = clock()): TopicStat[] {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND r.class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const rows = db.all<{ topic: string; class_id: string | null; class_name: string | null; class_color: string | null; seen: number; acc: number; questions: number }>(
`SELECT r.topic AS topic, r.class_id AS class_id, k.name AS class_name, k.color AS class_color,
COUNT(*) AS seen, AVG(r.score) AS acc,
(SELECT COUNT(*) FROM questions q JOIN quizzes z ON z.id = q.quiz_id WHERE q.topic = r.topic AND z.class_id = r.class_id) AS questions
FROM responses r LEFT JOIN classes k ON k.id = r.class_id
WHERE r.answered_at >= ? AND r.topic != '' ${cls}
GROUP BY r.topic, r.class_id
ORDER BY acc ASC, seen DESC`,
since,
...p,
);
return rows.map((r) => ({
topic: r.topic,
classId: r.class_id,
className: r.class_name ?? '',
classColor: r.class_color ?? 'slate',
seen: r.seen,
accuracy: r.acc,
questions: r.questions,
}));
}
export function weakTopics(db: DB, scope: { classId?: string }, limit = 6): TopicStat[] {
return topicStats(db, scope)
.filter((t) => t.seen >= 3 && t.accuracy < 0.75 && t.questions > 0)
.slice(0, limit);
}
export interface QuestionStat {
seen: number;
correct: number;
lastCorrect: boolean;
}
export function questionStats(db: DB, quizId: string): Record<string, QuestionStat> {
const rows = db.all<{ question_id: string; seen: number; correct: number; last_correct: number }>(
`SELECT r.question_id, COUNT(*) AS seen, SUM(r.correct) AS correct,
(SELECT r2.correct FROM responses r2 WHERE r2.question_id = r.question_id ORDER BY r2.answered_at DESC LIMIT 1) AS last_correct
FROM responses r JOIN questions q ON q.id = r.question_id WHERE q.quiz_id = ? GROUP BY r.question_id`,
quizId,
);
return Object.fromEntries(rows.map((r) => [r.question_id, { seen: r.seen, correct: r.correct, lastCorrect: Boolean(r.last_correct) }]));
}
export interface AccuracyPoint {
date: string;
answered: number;
accuracy: number | null;
}
export function accuracyTrend(db: DB, settings: AppSettings, days: number, scope: { classId?: string } = {}, at = clock()): AccuracyPoint[] {
const tz = settings.timezone;
const since = startOfDay(at, tz) - days * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const sums = new Map<string, { n: number; s: number }>();
for (const r of db.all<{ t: number; score: number }>(`SELECT answered_at AS t, score FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)) {
const key = dayKey(r.t, tz);
const cur = sums.get(key) ?? { n: 0, s: 0 };
cur.n += 1;
cur.s += r.score;
sums.set(key, cur);
}
const today = dayKey(at, tz);
const out: AccuracyPoint[] = [];
for (let i = days - 1; i >= 0; i -= 1) {
const key = addDaysToKey(today, -i);
const v = sums.get(key);
out.push({ date: key, answered: v?.n ?? 0, accuracy: v ? v.s / v.n : null });
}
return out;
}
export interface Totals {
questionsAnswered: number;
accuracy: number | null;
cardsReviewed: number;
retention: number | null;
quizzesCompleted: number;
studyDays: number;
}
export function totals(db: DB, settings: AppSettings, scope: { classId?: string } = {}, windowDays = 30, at = clock()): Totals {
const since = at - windowDays * DAY_MS;
const cls = scope.classId ? 'AND class_id = ?' : '';
const p = scope.classId ? [scope.classId] : [];
const resp = db.get<{ n: number; acc: number | null }>(`SELECT COUNT(*) AS n, AVG(score) AS acc FROM responses WHERE answered_at >= ? ${cls}`, since, ...p)!;
const rev = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM reviews WHERE reviewed_at >= ? ${cls}`, since, ...p)!;
// Retention: share of reviews of already-learned cards that were remembered (not "Again").
const ret = db.get<{ n: number; ok: number | null }>(
`SELECT COUNT(*) AS n, SUM(CASE WHEN rating > 1 THEN 1 ELSE 0 END) AS ok FROM reviews WHERE reviewed_at >= ? AND mode = 'srs' AND state_before = 2 ${cls}`,
since,
...p,
)!;
const done = db.get<{ n: number }>(`SELECT COUNT(*) AS n FROM attempts WHERE status = 'completed' AND completed_at >= ? ${cls}`, since, ...p)!;
const days = activityByDay(db, settings, windowDays, scope, at).filter((d) => d.total > 0).length;
return {
questionsAnswered: resp.n,
accuracy: resp.n > 0 ? resp.acc : null,
cardsReviewed: rev.n,
retention: ret.n >= 5 ? (ret.ok ?? 0) / ret.n : null,
quizzesCompleted: done.n,
studyDays: days,
};
}

42
server/settings.ts Normal file
View file

@ -0,0 +1,42 @@
import { z } from 'zod';
import type { DB } from './db.ts';
import { parseJson } from './util.ts';
export function getSetting<T>(db: DB, key: string, fallback: T): T {
const row = db.get<{ value: string }>('SELECT value FROM settings WHERE key = ?', key);
return row ? parseJson<T>(row.value, fallback) : fallback;
}
export function setSetting(db: DB, key: string, value: unknown): void {
db.run('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', key, JSON.stringify(value));
}
export function deleteSetting(db: DB, key: string): void {
db.run('DELETE FROM settings WHERE key = ?', key);
}
export const appSettingsSchema = z.object({
timezone: z.string().min(1).max(64),
/** New flashcards introduced per day across all decks. */
newCardsPerDay: z.number().int().min(0).max(500),
/** Target recall probability used by FSRS (0.7 - 0.97). */
desiredRetention: z.number().min(0.7).max(0.97),
sataScoring: z.enum(['partial', 'strict']),
/** Cards + questions per day that count as "studied" for the streak. */
dailyGoal: z.number().int().min(1).max(1000),
});
export type AppSettings = z.infer<typeof appSettingsSchema>;
export function getAppSettings(db: DB, defaultTimezone: string): AppSettings {
const stored = getSetting<Partial<AppSettings>>(db, 'app', {});
const merged = {
timezone: defaultTimezone,
newCardsPerDay: 20,
desiredRetention: 0.9,
sataScoring: 'partial' as const,
dailyGoal: 20,
...stored,
};
const parsed = appSettingsSchema.safeParse(merged);
return parsed.success ? parsed.data : { timezone: defaultTimezone, newCardsPerDay: 20, desiredRetention: 0.9, sataScoring: 'partial', dailyGoal: 20 };
}

97
server/util.ts Normal file
View file

@ -0,0 +1,97 @@
import crypto from 'node:crypto';
/** 16-character URL-safe id (96 bits). */
export function newId(): string {
return crypto.randomBytes(12).toString('base64url');
}
/** 32-character URL-safe token (192 bits) for share links and sessions. */
export function newToken(): string {
return crypto.randomBytes(24).toString('base64url');
}
export function sha256(input: string | Uint8Array): string {
return crypto.createHash('sha256').update(input).digest('hex');
}
export const now = (): number => Date.now();
export function parseJson<T>(text: string | null | undefined, fallback: T): T {
if (!text) return fallback;
try {
return JSON.parse(text) as T;
} catch {
return fallback;
}
}
export const DAY_MS = 86_400_000;
/** YYYY-MM-DD for a timestamp in the given IANA timezone. */
export function dayKey(ts: number, timeZone: string): string {
const parts = new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(new Date(ts));
const get = (t: string) => parts.find((p) => p.type === t)?.value ?? '';
return `${get('year')}-${get('month')}-${get('day')}`;
}
/** Offset in ms between UTC and the timezone at the given instant (positive = ahead of UTC). */
export function tzOffsetMs(ts: number, timeZone: string): number {
const parts = new Intl.DateTimeFormat('en-US', {
timeZone,
hourCycle: 'h23',
year: 'numeric',
month: 'numeric',
day: 'numeric',
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
}).formatToParts(new Date(ts));
const get = (t: string) => Number(parts.find((p) => p.type === t)?.value ?? 0);
const asUtc = Date.UTC(get('year'), get('month') - 1, get('day'), get('hour'), get('minute'), get('second'));
return asUtc - Math.floor(ts / 1000) * 1000;
}
/** Timestamp of local midnight that starts the day containing `ts` in the timezone. */
export function startOfDay(ts: number, timeZone: string): number {
const key = dayKey(ts, timeZone);
const utcMidnight = Date.parse(`${key}T00:00:00.000Z`);
// Two passes handle days where the offset at midnight differs from the offset at `ts` (DST).
let guess = utcMidnight - tzOffsetMs(ts, timeZone);
guess = utcMidnight - tzOffsetMs(guess, timeZone);
return guess;
}
export function addDaysToKey(key: string, days: number): string {
const d = new Date(`${key}T00:00:00.000Z`);
d.setUTCDate(d.getUTCDate() + days);
return d.toISOString().slice(0, 10);
}
export function clamp(n: number, min: number, max: number): number {
return Math.min(max, Math.max(min, n));
}
export function slugify(input: string): string {
return (
input
.toLowerCase()
.normalize('NFKD')
.replace(/[^\w\s-]/g, '')
.trim()
.replace(/[\s_]+/g, '-')
.slice(0, 60) || 'export'
);
}
/** Builds an FTS5 MATCH expression from free text: quoted terms OR-ed together, prefix match on the last term. */
export function ftsQuery(input: string, mode: 'or' | 'and' = 'or'): string | null {
const terms = (input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []).filter((t) => !STOP_WORDS.has(t)).slice(0, 24);
if (terms.length === 0) return null;
return [...new Set(terms)].map((t) => `"${t}"`).join(mode === 'or' ? ' OR ' : ' AND ');
}
const STOP_WORDS = new Set(
'a an and are as at be but by for from has have how if in into is it its of on or that the their them then there these this to was what when where which who why will with about your you can do does i me my we our should would could not no nor so than too very'.split(
' ',
),
);