Add drag reordering, exam deadlines, frozen shared quizzes, batched OCR and semantic search
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
- Drag-to-reorder for class materials, folders, quiz questions and deck cards. Pointer-based so touch works, plus keyboard (arrows/Home/End) with screen-reader announcements. - Timed exams get a wall-clock deadline. The server refuses late answers, submits overdue exams with what was saved, and the countdown keeps running while the page is closed. - Shared quiz sessions keep a frozen copy of their questions and flag later owner edits. - Scanned PDFs are read in cached 8-page batches with real page numbers (pdf-lib), removing the 30 MB whole-file limit. Compatible providers now receive PDFs as file parts. - Semantic search: embeddings for OpenAI, Gemini, Ollama and compatible servers, background indexing into chunk_vectors (migration 2), hybrid keyword + vector ranking for search, the assistant and focused generation, and a Settings -> AI section. 181 tests pass locally and in the Docker test stage on Node 24. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3046e46d1c
commit
ccdad9728b
39 changed files with 1772 additions and 121 deletions
|
|
@ -6,7 +6,7 @@
|
|||
import { z } from 'zod';
|
||||
import { isFullyCorrect, scoreQuestion, seededShuffle, type Option, type QuestionType, type SataScoring } from '../../shared/content.ts';
|
||||
import type { DB } from '../db.ts';
|
||||
import { badRequest, notFound } from '../http.ts';
|
||||
import { HttpError, badRequest, notFound } from '../http.ts';
|
||||
import { newId, now, parseJson } from '../util.ts';
|
||||
import { getClass, getQuiz, mapQuestion, type QuestionRow } from './content.ts';
|
||||
|
||||
|
|
@ -34,8 +34,19 @@ export interface AttemptSettings {
|
|||
shuffleOptions: boolean;
|
||||
timeLimitSec: number | null;
|
||||
sataScoring: SataScoring;
|
||||
/**
|
||||
* Wall-clock end of a timed session (epoch ms), fixed when it starts. The clock keeps
|
||||
* running while the session is closed; answers after it are refused and the session is
|
||||
* submitted with what was saved. Sessions from before this field use the active clock.
|
||||
*/
|
||||
deadlineAt?: number | null;
|
||||
}
|
||||
|
||||
/** Saves sent a few seconds after the deadline (network delay, the final autosave) still count. */
|
||||
export const DEADLINE_GRACE_MS = 5_000;
|
||||
|
||||
const DEFAULT_SETTINGS: AttemptSettings = { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' };
|
||||
|
||||
interface AttemptRow {
|
||||
id: string;
|
||||
quiz_id: string | null;
|
||||
|
|
@ -207,14 +218,16 @@ export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataSco
|
|||
if (opts.count && opts.count < questions.length) questions = questions.slice(0, opts.count);
|
||||
if (opts.shuffleOptions) questions = questions.map((q, i) => ({ ...q, options: shuffleOptions(q.options, seed + i + 1) }));
|
||||
|
||||
const id = newId();
|
||||
const at = now();
|
||||
const timeLimitSec = opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null;
|
||||
const settings: AttemptSettings = {
|
||||
shuffleQuestions: opts.shuffleQuestions,
|
||||
shuffleOptions: opts.shuffleOptions,
|
||||
timeLimitSec: opts.timeLimitMin ? Math.round(opts.timeLimitMin * 60) : null,
|
||||
timeLimitSec,
|
||||
sataScoring,
|
||||
deadlineAt: timeLimitSec ? at + timeLimitSec * 1000 : null,
|
||||
};
|
||||
const id = newId();
|
||||
const at = now();
|
||||
db.run(
|
||||
`INSERT INTO attempts (id, quiz_id, class_id, title, mode, kind, status, settings, snapshot, question_count, started_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)`,
|
||||
|
|
@ -233,12 +246,38 @@ export function createAttempt(db: DB, input: AttemptCreate, sataScoring: SataSco
|
|||
return id;
|
||||
}
|
||||
|
||||
function loadAttempt(db: DB, id: string): AttemptRow {
|
||||
function loadRaw(db: DB, id: string): AttemptRow {
|
||||
const row = db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', id);
|
||||
if (!row) throw notFound('Session');
|
||||
return row;
|
||||
}
|
||||
|
||||
/** Loads a session, first submitting it if its deadline has passed. */
|
||||
function loadAttempt(db: DB, id: string): AttemptRow {
|
||||
const row = loadRaw(db, id);
|
||||
if (isOverdue(row)) {
|
||||
completeAttempt(db, row.id);
|
||||
return db.get<AttemptRow>('SELECT * FROM attempts WHERE id = ?', id)!;
|
||||
}
|
||||
return row;
|
||||
}
|
||||
|
||||
function isOverdue(row: AttemptRow, at = now()): boolean {
|
||||
if (row.status !== 'active') return false;
|
||||
const deadline = parseJson<AttemptSettings>(row.settings, DEFAULT_SETTINGS).deadlineAt;
|
||||
return typeof deadline === 'number' && at > deadline + DEADLINE_GRACE_MS;
|
||||
}
|
||||
|
||||
/** Submits every timed session whose deadline has passed. Returns how many were submitted. */
|
||||
export function expireOverdueAttempts(db: DB): number {
|
||||
const rows = db.all<{ id: string }>(
|
||||
`SELECT id FROM attempts WHERE status = 'active' AND json_extract(settings, '$.deadlineAt') IS NOT NULL AND json_extract(settings, '$.deadlineAt') < ?`,
|
||||
now() - DEADLINE_GRACE_MS,
|
||||
);
|
||||
for (const r of rows) completeAttempt(db, r.id);
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected: string[], sata: SataScoring, at: number): void {
|
||||
const score = scoreQuestion(q, selected, sata);
|
||||
db.run(
|
||||
|
|
@ -256,7 +295,7 @@ function recordResponse(db: DB, row: AttemptRow, q: SnapshotQuestion, selected:
|
|||
}
|
||||
|
||||
export function mapAttempt(row: AttemptRow) {
|
||||
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
|
||||
const settings = parseJson<AttemptSettings>(row.settings, DEFAULT_SETTINGS);
|
||||
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
|
||||
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
|
||||
const results =
|
||||
|
|
@ -289,6 +328,8 @@ export function mapAttempt(row: AttemptRow) {
|
|||
updatedAt: row.updated_at,
|
||||
completedAt: row.completed_at,
|
||||
results,
|
||||
/** Server clock when this was produced, so clients can correct for clock skew. */
|
||||
serverNow: now(),
|
||||
};
|
||||
}
|
||||
export type AttemptDto = ReturnType<typeof mapAttempt>;
|
||||
|
|
@ -331,10 +372,12 @@ export const attemptPatchSchema = z.object({
|
|||
|
||||
export function patchAttempt(db: DB, id: string, input: z.input<typeof attemptPatchSchema>): AttemptDto {
|
||||
const patch = attemptPatchSchema.parse(input);
|
||||
const overdue = isOverdue(loadRaw(db, id));
|
||||
const row = loadAttempt(db, id);
|
||||
if (overdue) throw new HttpError(409, 'time_up', 'Time is up. The exam was submitted with the answers saved before the deadline.');
|
||||
if (row.status !== 'active') throw badRequest('This session is already finished');
|
||||
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
|
||||
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
|
||||
const settings = parseJson<AttemptSettings>(row.settings, DEFAULT_SETTINGS);
|
||||
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
|
||||
const at = now();
|
||||
db.tx(() => {
|
||||
|
|
@ -371,12 +414,17 @@ export function patchAttempt(db: DB, id: string, input: z.input<typeof attemptPa
|
|||
* ended early are scored on the questions actually answered.
|
||||
*/
|
||||
export function completeAttempt(db: DB, id: string, elapsedSec?: number): AttemptDto {
|
||||
const row = loadAttempt(db, id);
|
||||
const row = loadRaw(db, id);
|
||||
if (row.status === 'completed') return mapAttempt(row);
|
||||
const snapshot = parseJson<{ questions: SnapshotQuestion[] }>(row.snapshot, { questions: [] });
|
||||
const settings = parseJson<AttemptSettings>(row.settings, { shuffleQuestions: true, shuffleOptions: true, timeLimitSec: null, sataScoring: 'partial' });
|
||||
const settings = parseJson<AttemptSettings>(row.settings, DEFAULT_SETTINGS);
|
||||
const answers = parseJson<Record<string, AnswerState>>(row.answers, {});
|
||||
const at = now();
|
||||
let elapsed = elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec;
|
||||
if (settings.deadlineAt && settings.timeLimitSec) {
|
||||
// Timed sessions run on the wall clock: time used is time since the start, capped at the limit.
|
||||
elapsed = Math.min(settings.timeLimitSec, Math.max(0, Math.round((at - row.started_at) / 1000)));
|
||||
}
|
||||
let score = 0;
|
||||
let max = 0;
|
||||
db.tx(() => {
|
||||
|
|
@ -395,7 +443,7 @@ export function completeAttempt(db: DB, id: string, elapsedSec?: number): Attemp
|
|||
max,
|
||||
at,
|
||||
at,
|
||||
elapsedSec !== undefined ? Math.max(row.elapsed_sec, elapsedSec) : row.elapsed_sec,
|
||||
elapsed,
|
||||
id,
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@ import { AIError, type ContentPart } from '../ai/types.ts';
|
|||
import type { DB } from '../db.ts';
|
||||
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
|
||||
import { detectKind, extractDocument } from '../documents/extract.ts';
|
||||
import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts';
|
||||
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
|
||||
import { HttpError, notFound, type AppContext } from '../http.ts';
|
||||
import { ftsQuery, newId, now, parseJson } from '../util.ts';
|
||||
import { getClass, resolveFolder } from './content.ts';
|
||||
import { scheduleIndexing } from './semantic.ts';
|
||||
|
||||
export interface DocumentRow {
|
||||
id: string;
|
||||
|
|
@ -150,6 +152,7 @@ export function deleteDocument(app: AppContext, id: string): void {
|
|||
app.db.run('DELETE FROM documents WHERE id = ?', id);
|
||||
});
|
||||
fs.rmSync(documentPath(app, row), { force: true });
|
||||
fs.rmSync(ocrCachePath(app, row), { force: true });
|
||||
}
|
||||
|
||||
/* ------------------------------ processing queue ------------------------------ */
|
||||
|
|
@ -208,43 +211,136 @@ const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
|
|||
- If something is illegible, write [illegible].
|
||||
Return only the transcription.`;
|
||||
|
||||
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
|
||||
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
|
||||
const data = Buffer.from(bytes).toString('base64');
|
||||
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
|
||||
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
|
||||
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
|
||||
/** Long scanned PDFs are read this many pages at a time. */
|
||||
export const OCR_PAGES_PER_BATCH = 8;
|
||||
/** Largest single request sent to a model (providers cap inline files around 32 MB). */
|
||||
const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024;
|
||||
/** Upper bound on pages read with AI for one document, to keep costs predictable. */
|
||||
export const OCR_MAX_PAGES = 600;
|
||||
|
||||
interface OcrCache {
|
||||
modelId: string;
|
||||
/** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */
|
||||
batches: Record<string, ExtractedSection[]>;
|
||||
}
|
||||
|
||||
const ocrCachePath = (app: AppContext, row: Pick<DocumentRow, 'storage_name'>) => `${documentPath(app, row)}.ocr.json`;
|
||||
|
||||
/** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */
|
||||
export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] {
|
||||
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
|
||||
if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : [];
|
||||
const sections: ExtractedSection[] = [];
|
||||
if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage });
|
||||
for (let i = 1; i < pages.length; i += 2) {
|
||||
const body = (pages[i + 1] ?? '').trim();
|
||||
let page = Number(pages[i]);
|
||||
// Models sometimes number a batch from 1; map those back onto the batch's real pages.
|
||||
if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1;
|
||||
if (body) sections.push({ text: body, page });
|
||||
}
|
||||
let result;
|
||||
return sections;
|
||||
}
|
||||
|
||||
interface Transcription {
|
||||
sections: ExtractedSection[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a
|
||||
* few pages, with finished batches cached beside the upload so a restart or a failed batch does
|
||||
* not repeat paid work. Returns null when no vision model is configured.
|
||||
*/
|
||||
async function transcribeWithVision(
|
||||
app: AppContext,
|
||||
row: DocumentRow,
|
||||
bytes: Uint8Array,
|
||||
opts: { fresh: boolean; onProgress?: (fraction: number) => void },
|
||||
): Promise<Transcription | null> {
|
||||
let vision;
|
||||
try {
|
||||
const vision = resolveModel(app, 'vision');
|
||||
if (!vision.model.capabilities.vision) return null;
|
||||
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
|
||||
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
|
||||
}
|
||||
result = await runChat(app, {
|
||||
task: 'vision',
|
||||
purpose: 'document transcription',
|
||||
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
|
||||
maxOutputTokens: 32000,
|
||||
reasoningEffort: 'low',
|
||||
});
|
||||
vision = resolveModel(app, 'vision');
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'not_configured') return null;
|
||||
throw err;
|
||||
}
|
||||
const text = result.text.trim();
|
||||
if (!text) return [];
|
||||
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
|
||||
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
|
||||
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
|
||||
const sections: ExtractedSection[] = [];
|
||||
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
|
||||
for (let i = 1; i < pages.length; i += 2) {
|
||||
const body = (pages[i + 1] ?? '').trim();
|
||||
if (body) sections.push({ text: body, page: Number(pages[i]) });
|
||||
if (!vision.model.capabilities.vision) return null;
|
||||
|
||||
const ask = async (part: ContentPart, prompt: string) => {
|
||||
const result = await runChat(app, {
|
||||
task: 'vision',
|
||||
purpose: 'document transcription',
|
||||
messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }],
|
||||
maxOutputTokens: 32000,
|
||||
reasoningEffort: 'low',
|
||||
});
|
||||
return result.text.trim();
|
||||
};
|
||||
|
||||
if (row.kind === 'image') {
|
||||
const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT);
|
||||
return { sections: text ? [{ text }] : [], warnings: [] };
|
||||
}
|
||||
return sections;
|
||||
|
||||
if (!vision.model.capabilities.pdf) {
|
||||
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
|
||||
}
|
||||
|
||||
const cacheFile = ocrCachePath(app, row);
|
||||
let cache: OcrCache = { modelId: vision.model.id, batches: {} };
|
||||
if (opts.fresh) fs.rmSync(cacheFile, { force: true });
|
||||
else {
|
||||
try {
|
||||
const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache;
|
||||
if (saved.modelId === vision.model.id && saved.batches) cache = saved;
|
||||
} catch {
|
||||
/* no cache yet */
|
||||
}
|
||||
}
|
||||
const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache));
|
||||
|
||||
const warnings: string[] = [];
|
||||
let total: number;
|
||||
try {
|
||||
total = await pdfPageCount(bytes);
|
||||
} catch {
|
||||
throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.');
|
||||
}
|
||||
const limit = Math.min(total, OCR_MAX_PAGES);
|
||||
if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`);
|
||||
|
||||
const sections: ExtractedSection[] = [];
|
||||
for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) {
|
||||
if (batch.start > limit) break;
|
||||
const key = `${batch.start}-${batch.end}`;
|
||||
const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`;
|
||||
let got: ExtractedSection[] | undefined = cache.batches[key];
|
||||
if (!got) {
|
||||
if (batch.bytes.length > OCR_MAX_BATCH_BYTES) {
|
||||
warnings.push(`${label} is too large to send to an AI model and was skipped.`);
|
||||
} else {
|
||||
const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`;
|
||||
const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` };
|
||||
for (let attempt = 1; attempt <= 2 && !got; attempt++) {
|
||||
try {
|
||||
got = splitTranscription(await ask(part, prompt), batch.start, batch);
|
||||
} catch (err) {
|
||||
// Configuration problems will not fix themselves on the next batch.
|
||||
if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err;
|
||||
if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`);
|
||||
}
|
||||
}
|
||||
if (got) {
|
||||
cache.batches[key] = got;
|
||||
save();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (got) sections.push(...got);
|
||||
opts.onProgress?.(Math.min(limit, batch.end) / limit);
|
||||
}
|
||||
return { sections, warnings };
|
||||
}
|
||||
|
||||
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
|
||||
|
|
@ -279,15 +375,16 @@ export async function processDocument(app: AppContext, id: string, opts: { force
|
|||
setProgress(0.82);
|
||||
if (row.kind === 'image' || row.kind === 'pdf') {
|
||||
try {
|
||||
const transcribed = await transcribeWithVision(app, row, bytes);
|
||||
const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) });
|
||||
if (transcribed) warnings.push(...transcribed.warnings);
|
||||
if (transcribed === null) {
|
||||
warnings.push(
|
||||
row.kind === 'image'
|
||||
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
|
||||
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
|
||||
);
|
||||
} else if (transcribed.length > 0) {
|
||||
sections = transcribed;
|
||||
} else if (transcribed.sections.length > 0) {
|
||||
sections = transcribed.sections;
|
||||
needsOcr = false;
|
||||
} else {
|
||||
warnings.push('The AI model did not find any readable text in this file.');
|
||||
|
|
@ -325,11 +422,12 @@ export async function processDocument(app: AppContext, id: string, opts: { force
|
|||
tokens,
|
||||
chunks.length,
|
||||
needsOcr ? 1 : 0,
|
||||
JSON.stringify([...new Set(warnings)].slice(0, 8)),
|
||||
JSON.stringify([...new Set(warnings)].slice(0, 12)),
|
||||
now(),
|
||||
id,
|
||||
);
|
||||
});
|
||||
scheduleIndexing(app);
|
||||
} catch (err) {
|
||||
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
|
||||
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
|
||||
|
|
@ -432,7 +530,7 @@ export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' |
|
|||
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
|
||||
* so every part of the document is represented.
|
||||
*/
|
||||
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
|
||||
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number; ranked?: ChunkHit[] }): ChunkHit[][] {
|
||||
const { sql, params } = scopeSql(scope);
|
||||
if (!sql) return [];
|
||||
const all = db
|
||||
|
|
@ -448,7 +546,8 @@ export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string;
|
|||
let chosen = all;
|
||||
|
||||
if (opts.focus?.trim()) {
|
||||
const ranked = searchChunks(db, opts.focus, scope, 400);
|
||||
// Callers pass a hybrid (keyword + semantic) ranking when they have one.
|
||||
const ranked = opts.ranked ?? searchChunks(db, opts.focus, scope, 400);
|
||||
if (ranked.length >= 3) {
|
||||
const keep = new Set<number>();
|
||||
let used = 0;
|
||||
|
|
|
|||
320
server/services/semantic.ts
Normal file
320
server/services/semantic.ts
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
/**
|
||||
* Semantic search over document passages.
|
||||
*
|
||||
* Each chunk gets an embedding from the configured provider (OpenAI, Gemini, Ollama or any
|
||||
* OpenAI-compatible server). Vectors are normalised Float32 BLOBs in `chunk_vectors`, keyed by
|
||||
* provider + model + size, and searched by brute-force cosine similarity over an in-memory
|
||||
* copy: a personal library has thousands of passages, not millions, so no index is needed.
|
||||
*
|
||||
* Search is hybrid: keyword (FTS5 BM25) and semantic rankings are merged with reciprocal-rank
|
||||
* fusion, so exact terms (drug names, lab values) still win while paraphrased questions
|
||||
* ("why do patients on furosemide need bananas?") also find the right passage.
|
||||
*/
|
||||
import { z } from 'zod';
|
||||
import { getAdapter } from '../ai/providers/index.ts';
|
||||
import { providerConfigById } from '../ai/service.ts';
|
||||
import { AIError, type ProviderKind } from '../ai/types.ts';
|
||||
import type { DB } from '../db.ts';
|
||||
import { badRequest, type AppContext } from '../http.ts';
|
||||
import { getSetting, setSetting } from '../settings.ts';
|
||||
import { now } from '../util.ts';
|
||||
import { chunkById, searchChunks, type ChunkHit, type ChunkScope } from './documents.ts';
|
||||
|
||||
/* ------------------------------ settings ------------------------------ */
|
||||
|
||||
export interface EmbeddingSettings {
|
||||
providerId: string | null;
|
||||
model: string;
|
||||
/** Requested vector size for models that support shortening (OpenAI v3, Gemini). Null = native. */
|
||||
dimensions: number | null;
|
||||
}
|
||||
|
||||
const SETTINGS_KEY = 'embeddings';
|
||||
const EMPTY: EmbeddingSettings = { providerId: null, model: '', dimensions: null };
|
||||
|
||||
/** Suggested embedding model per provider type. Anthropic has no embeddings API. */
|
||||
export const DEFAULT_EMBEDDING_MODELS: Record<ProviderKind, string | null> = {
|
||||
openai: 'text-embedding-3-small',
|
||||
gemini: 'gemini-embedding-001',
|
||||
ollama: 'nomic-embed-text',
|
||||
openai_compatible: '',
|
||||
anthropic: null,
|
||||
};
|
||||
|
||||
/** USD per million input tokens, for the usage log. */
|
||||
const PRICES: Record<string, number> = {
|
||||
'text-embedding-3-small': 0.02,
|
||||
'text-embedding-3-large': 0.13,
|
||||
'text-embedding-ada-002': 0.1,
|
||||
'gemini-embedding-001': 0.15,
|
||||
};
|
||||
|
||||
export const embeddingSettingsSchema = z.object({
|
||||
providerId: z.string().nullable(),
|
||||
model: z.string().trim().max(200).default(''),
|
||||
dimensions: z.number().int().min(64).max(4096).nullable().default(null),
|
||||
});
|
||||
|
||||
export function getEmbeddingSettings(db: DB): EmbeddingSettings {
|
||||
return { ...EMPTY, ...getSetting<Partial<EmbeddingSettings>>(db, SETTINGS_KEY, EMPTY) };
|
||||
}
|
||||
|
||||
const modelKey = (s: EmbeddingSettings) => `${s.providerId}:${s.model}:${s.dimensions ?? 'native'}`;
|
||||
const isConfigured = (s: EmbeddingSettings) => Boolean(s.providerId && s.model);
|
||||
|
||||
export function saveEmbeddingSettings(app: AppContext, input: z.input<typeof embeddingSettingsSchema>): EmbeddingSettings {
|
||||
const next = embeddingSettingsSchema.parse(input);
|
||||
if (next.providerId) {
|
||||
const { kind } = providerConfigById(app, next.providerId);
|
||||
if (!getAdapter(kind).embed) throw badRequest('This provider has no embeddings API. Choose OpenAI, Gemini, Ollama or an OpenAI-compatible server.');
|
||||
if (!next.model) throw badRequest('Enter the embedding model to use.');
|
||||
}
|
||||
const prev = getEmbeddingSettings(app.db);
|
||||
setSetting(app.db, SETTINGS_KEY, next);
|
||||
if (modelKey(prev) !== modelKey(next)) {
|
||||
// Vectors from different models are not comparable: start over.
|
||||
app.db.run('DELETE FROM chunk_vectors WHERE model_key != ?', modelKey(next));
|
||||
cache = null;
|
||||
state.lastError = '';
|
||||
}
|
||||
if (isConfigured(next)) scheduleIndexing(app);
|
||||
return next;
|
||||
}
|
||||
|
||||
/* ------------------------------ embedding calls ------------------------------ */
|
||||
|
||||
/** Inputs per request and a cap on characters per input (about 2k tokens). */
|
||||
const BATCH = 64;
|
||||
const MAX_INPUT_CHARS = 8000;
|
||||
|
||||
function normalise(v: number[]): Float32Array {
|
||||
const out = new Float32Array(v.length);
|
||||
let sum = 0;
|
||||
for (const x of v) sum += x * x;
|
||||
const len = Math.sqrt(sum) || 1;
|
||||
for (let i = 0; i < v.length; i++) out[i] = v[i] / len;
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function embedTexts(app: AppContext, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]> {
|
||||
const s = getEmbeddingSettings(app.db);
|
||||
if (!isConfigured(s)) throw new AIError('not_configured', 'Semantic search is not set up. Choose an embedding model in Settings → AI.');
|
||||
const { cfg, kind, name } = providerConfigById(app, s.providerId!);
|
||||
const adapter = getAdapter(kind);
|
||||
if (!adapter.embed) throw new AIError('unsupported', `${name} has no embeddings API.`);
|
||||
const out: Float32Array[] = [];
|
||||
for (let i = 0; i < texts.length; i += BATCH) {
|
||||
const inputs = texts.slice(i, i + BATCH).map((t) => t.slice(0, MAX_INPUT_CHARS) || ' ');
|
||||
const started = now();
|
||||
try {
|
||||
const res = await adapter.embed(cfg, { model: s.model, inputs, purpose, dimensions: s.dimensions ?? undefined, signal });
|
||||
out.push(...res.vectors.map(normalise));
|
||||
logUsage(app.db, s, name, purpose, res.inputTokens, started);
|
||||
} catch (err) {
|
||||
logUsage(app.db, s, name, purpose, 0, started, err instanceof Error ? err.message : String(err));
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function logUsage(db: DB, s: EmbeddingSettings, providerName: string, purpose: string, tokens: number, started: number, error?: string) {
|
||||
try {
|
||||
const price = PRICES[s.model.replace(/^models\//, '')];
|
||||
db.run(
|
||||
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, 0, 0, ?, ?, ?, ?)`,
|
||||
now(),
|
||||
s.providerId,
|
||||
providerName,
|
||||
s.model,
|
||||
purpose === 'query' ? 'semantic search' : 'semantic indexing',
|
||||
tokens,
|
||||
price !== undefined ? (tokens * price) / 1_000_000 : null,
|
||||
error ? 0 : 1,
|
||||
error ? error.slice(0, 500) : '',
|
||||
now() - started,
|
||||
);
|
||||
} catch {
|
||||
/* usage logging must never break search */
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------ background indexing ------------------------------ */
|
||||
|
||||
const state = { running: false, lastError: '', lastRunAt: 0 };
|
||||
|
||||
/** Embeds every chunk that has no vector for the current model. Safe to call often. */
|
||||
export function scheduleIndexing(app: AppContext): void {
|
||||
if (state.running || !isConfigured(getEmbeddingSettings(app.db))) return;
|
||||
state.running = true;
|
||||
state.lastError = '';
|
||||
void (async () => {
|
||||
try {
|
||||
for (;;) {
|
||||
const s = getEmbeddingSettings(app.db);
|
||||
if (!isConfigured(s)) break;
|
||||
const key = modelKey(s);
|
||||
const rows = app.db.all<{ id: number; text: string; heading: string; title: string }>(
|
||||
`SELECT ch.id, ch.text, ch.heading, d.title FROM chunks ch JOIN documents d ON d.id = ch.document_id
|
||||
LEFT JOIN chunk_vectors v ON v.chunk_id = ch.id AND v.model_key = ?
|
||||
WHERE v.chunk_id IS NULL ORDER BY ch.id LIMIT ?`,
|
||||
key,
|
||||
BATCH,
|
||||
);
|
||||
if (rows.length === 0) break;
|
||||
// The document title and heading give a passage its context ("Digoxin" under "Cardiac drugs").
|
||||
const vectors = await embedTexts(app, rows.map((r) => [r.title, r.heading, r.text].filter(Boolean).join('\n')), 'document');
|
||||
if (modelKey(getEmbeddingSettings(app.db)) !== key) continue; // settings changed mid-batch
|
||||
app.db.tx(() => {
|
||||
rows.forEach((r, i) => {
|
||||
const v = vectors[i];
|
||||
// The chunk may have been deleted while it was being embedded.
|
||||
if (app.db.get('SELECT 1 FROM chunks WHERE id = ?', r.id)) {
|
||||
app.db.run('INSERT OR REPLACE INTO chunk_vectors (chunk_id, model_key, dims, vector) VALUES (?, ?, ?, ?)', r.id, key, v.length, new Uint8Array(v.buffer, v.byteOffset, v.byteLength));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
state.lastError = err instanceof Error ? err.message : String(err);
|
||||
app.log('warn', `Semantic indexing paused: ${state.lastError}`);
|
||||
} finally {
|
||||
state.running = false;
|
||||
state.lastRunAt = now();
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
/** Waits for the current indexing run. Used by tests. */
|
||||
export async function indexingIdle(): Promise<void> {
|
||||
while (state.running) await new Promise((r) => setTimeout(r, 10));
|
||||
}
|
||||
|
||||
export function rebuildIndex(app: AppContext): void {
|
||||
app.db.run('DELETE FROM chunk_vectors');
|
||||
cache = null;
|
||||
state.lastError = '';
|
||||
scheduleIndexing(app);
|
||||
}
|
||||
|
||||
export function embeddingStatus(app: AppContext) {
|
||||
const s = getEmbeddingSettings(app.db);
|
||||
const total = app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunks')?.n ?? 0;
|
||||
const indexed = isConfigured(s) ? (app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunk_vectors WHERE model_key = ?', modelKey(s))?.n ?? 0) : 0;
|
||||
return { settings: s, configured: isConfigured(s), indexed, total, running: state.running, lastError: state.lastError, defaults: DEFAULT_EMBEDDING_MODELS };
|
||||
}
|
||||
|
||||
/* ------------------------------ vector search ------------------------------ */
|
||||
|
||||
interface VectorCache {
|
||||
key: string;
|
||||
count: number;
|
||||
maxId: number;
|
||||
dims: number;
|
||||
ids: Int32Array;
|
||||
docIds: string[];
|
||||
classIds: string[];
|
||||
matrix: Float32Array;
|
||||
}
|
||||
let cache: VectorCache | null = null;
|
||||
|
||||
function loadVectors(db: DB, key: string): VectorCache | null {
|
||||
const sig = db.get<{ n: number; m: number | null }>('SELECT COUNT(*) AS n, MAX(chunk_id) AS m FROM chunk_vectors WHERE model_key = ?', key)!;
|
||||
if (sig.n === 0) return null;
|
||||
if (cache && cache.key === key && cache.count === sig.n && cache.maxId === (sig.m ?? 0)) return cache;
|
||||
const rows = db.all<{ chunk_id: number; document_id: string; class_id: string; dims: number; vector: Uint8Array }>(
|
||||
`SELECT v.chunk_id, ch.document_id, ch.class_id, v.dims, v.vector FROM chunk_vectors v JOIN chunks ch ON ch.id = v.chunk_id WHERE v.model_key = ?`,
|
||||
key,
|
||||
);
|
||||
const dims = rows[0]?.dims ?? 0;
|
||||
const usable = rows.filter((r) => r.dims === dims && r.vector.byteLength === dims * 4);
|
||||
const matrix = new Float32Array(usable.length * dims);
|
||||
usable.forEach((r, i) => matrix.set(new Float32Array(r.vector.slice().buffer), i * dims));
|
||||
cache = {
|
||||
key,
|
||||
count: sig.n,
|
||||
maxId: sig.m ?? 0,
|
||||
dims,
|
||||
ids: Int32Array.from(usable.map((r) => r.chunk_id)),
|
||||
docIds: usable.map((r) => r.document_id),
|
||||
classIds: usable.map((r) => r.class_id),
|
||||
matrix,
|
||||
};
|
||||
return cache;
|
||||
}
|
||||
|
||||
const queryCache = new Map<string, Float32Array>();
|
||||
|
||||
async function queryVector(app: AppContext, key: string, query: string): Promise<Float32Array> {
|
||||
const k = `${key}\n${query}`;
|
||||
const hit = queryCache.get(k);
|
||||
if (hit) return hit;
|
||||
const [v] = await embedTexts(app, [query], 'query');
|
||||
queryCache.set(k, v);
|
||||
if (queryCache.size > 200) queryCache.delete(queryCache.keys().next().value!);
|
||||
return v;
|
||||
}
|
||||
|
||||
/** Passages closest in meaning to the query, best first. Empty when semantic search is off. */
|
||||
export async function semanticSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8, minScore = 0.2): Promise<(ChunkHit & { similarity: number })[]> {
|
||||
const s = getEmbeddingSettings(app.db);
|
||||
if (!isConfigured(s) || !query.trim()) return [];
|
||||
const key = modelKey(s);
|
||||
const vectors = loadVectors(app.db, key);
|
||||
if (!vectors) return [];
|
||||
const q = await queryVector(app, key, query.trim().slice(0, 2000));
|
||||
if (q.length !== vectors.dims) return [];
|
||||
const docs = scope.documentIds?.length ? new Set(scope.documentIds) : null;
|
||||
const top: { i: number; score: number }[] = [];
|
||||
const { dims, matrix } = vectors;
|
||||
for (let i = 0; i < vectors.ids.length; i++) {
|
||||
if (docs ? !docs.has(vectors.docIds[i]) : scope.classId && vectors.classIds[i] !== scope.classId) continue;
|
||||
let dot = 0;
|
||||
const off = i * dims;
|
||||
for (let d = 0; d < dims; d++) dot += matrix[off + d] * q[d];
|
||||
if (dot < minScore) continue;
|
||||
if (top.length < limit) {
|
||||
top.push({ i, score: dot });
|
||||
top.sort((a, b) => b.score - a.score);
|
||||
} else if (dot > top[top.length - 1].score) {
|
||||
top[top.length - 1] = { i, score: dot };
|
||||
top.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
}
|
||||
const out: (ChunkHit & { similarity: number })[] = [];
|
||||
for (const t of top) {
|
||||
const hit = chunkById(app.db, vectors.ids[t.i]);
|
||||
if (hit) out.push({ ...hit, similarity: t.score });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keyword + semantic search merged by reciprocal-rank fusion. Falls back to keyword search alone
|
||||
* when semantic search is off or its provider fails, so search never breaks.
|
||||
*/
|
||||
export async function hybridSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8): Promise<ChunkHit[]> {
|
||||
const pool = Math.max(limit * 3, 30);
|
||||
const lexical = searchChunks(app.db, query, scope, pool);
|
||||
let semantic: ChunkHit[] = [];
|
||||
try {
|
||||
semantic = await semanticSearch(app, query, scope, pool);
|
||||
} catch (err) {
|
||||
app.log('warn', `Semantic search unavailable, using keywords only: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
if (semantic.length === 0) return lexical.slice(0, limit);
|
||||
const K = 60;
|
||||
const fused = new Map<number, { hit: ChunkHit; score: number }>();
|
||||
const add = (list: ChunkHit[]) =>
|
||||
list.forEach((hit, rank) => {
|
||||
const prev = fused.get(hit.id);
|
||||
fused.set(hit.id, { hit: prev?.hit ?? hit, score: (prev?.score ?? 0) + 1 / (K + rank + 1) });
|
||||
});
|
||||
add(lexical);
|
||||
add(semantic);
|
||||
return [...fused.values()]
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
.map((x) => ({ ...x.hit, score: x.score }));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue