Add drag reordering, exam deadlines, frozen shared quizzes, batched OCR and semantic search
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s

- Drag-to-reorder for class materials, folders, quiz questions and deck cards. Pointer-based
  so touch works, plus keyboard (arrows/Home/End) with screen-reader announcements.
- Timed exams get a wall-clock deadline. The server refuses late answers, submits overdue
  exams with what was saved, and the countdown keeps running while the page is closed.
- Shared quiz sessions keep a frozen copy of their questions and flag later owner edits.
- Scanned PDFs are read in cached 8-page batches with real page numbers (pdf-lib), removing
  the 30 MB whole-file limit. Compatible providers now receive PDFs as file parts.
- Semantic search: embeddings for OpenAI, Gemini, Ollama and compatible servers, background
  indexing into chunk_vectors (migration 2), hybrid keyword + vector ranking for search, the
  assistant and focused generation, and a Settings -> AI section.

181 tests pass locally and in the Docker test stage on Node 24.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-12 11:48:20 -07:00
parent 3046e46d1c
commit ccdad9728b
39 changed files with 1772 additions and 121 deletions

View file

@ -10,10 +10,12 @@ import { AIError, type ContentPart } from '../ai/types.ts';
import type { DB } from '../db.ts';
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
import { detectKind, extractDocument } from '../documents/extract.ts';
import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts';
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { ftsQuery, newId, now, parseJson } from '../util.ts';
import { getClass, resolveFolder } from './content.ts';
import { scheduleIndexing } from './semantic.ts';
export interface DocumentRow {
id: string;
@ -150,6 +152,7 @@ export function deleteDocument(app: AppContext, id: string): void {
app.db.run('DELETE FROM documents WHERE id = ?', id);
});
fs.rmSync(documentPath(app, row), { force: true });
fs.rmSync(ocrCachePath(app, row), { force: true });
}
/* ------------------------------ processing queue ------------------------------ */
@ -208,43 +211,136 @@ const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
- If something is illegible, write [illegible].
Return only the transcription.`;
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
const data = Buffer.from(bytes).toString('base64');
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
/** Long scanned PDFs are read this many pages at a time. */
export const OCR_PAGES_PER_BATCH = 8;
/** Largest single request sent to a model (providers cap inline files around 32 MB). */
const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024;
/** Upper bound on pages read with AI for one document, to keep costs predictable. */
export const OCR_MAX_PAGES = 600;
interface OcrCache {
modelId: string;
/** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */
batches: Record<string, ExtractedSection[]>;
}
const ocrCachePath = (app: AppContext, row: Pick<DocumentRow, 'storage_name'>) => `${documentPath(app, row)}.ocr.json`;
/** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */
export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] {
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : [];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
let page = Number(pages[i]);
// Models sometimes number a batch from 1; map those back onto the batch's real pages.
if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1;
if (body) sections.push({ text: body, page });
}
let result;
return sections;
}
interface Transcription {
sections: ExtractedSection[];
warnings: string[];
}
/**
* Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a
* few pages, with finished batches cached beside the upload so a restart or a failed batch does
* not repeat paid work. Returns null when no vision model is configured.
*/
async function transcribeWithVision(
app: AppContext,
row: DocumentRow,
bytes: Uint8Array,
opts: { fresh: boolean; onProgress?: (fraction: number) => void },
): Promise<Transcription | null> {
let vision;
try {
const vision = resolveModel(app, 'vision');
if (!vision.model.capabilities.vision) return null;
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
vision = resolveModel(app, 'vision');
} catch (err) {
if (err instanceof AIError && err.code === 'not_configured') return null;
throw err;
}
const text = result.text.trim();
if (!text) return [];
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
if (body) sections.push({ text: body, page: Number(pages[i]) });
if (!vision.model.capabilities.vision) return null;
const ask = async (part: ContentPart, prompt: string) => {
const result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
return result.text.trim();
};
if (row.kind === 'image') {
const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT);
return { sections: text ? [{ text }] : [], warnings: [] };
}
return sections;
if (!vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
const cacheFile = ocrCachePath(app, row);
let cache: OcrCache = { modelId: vision.model.id, batches: {} };
if (opts.fresh) fs.rmSync(cacheFile, { force: true });
else {
try {
const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache;
if (saved.modelId === vision.model.id && saved.batches) cache = saved;
} catch {
/* no cache yet */
}
}
const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache));
const warnings: string[] = [];
let total: number;
try {
total = await pdfPageCount(bytes);
} catch {
throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.');
}
const limit = Math.min(total, OCR_MAX_PAGES);
if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`);
const sections: ExtractedSection[] = [];
for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) {
if (batch.start > limit) break;
const key = `${batch.start}-${batch.end}`;
const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`;
let got: ExtractedSection[] | undefined = cache.batches[key];
if (!got) {
if (batch.bytes.length > OCR_MAX_BATCH_BYTES) {
warnings.push(`${label} is too large to send to an AI model and was skipped.`);
} else {
const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`;
const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` };
for (let attempt = 1; attempt <= 2 && !got; attempt++) {
try {
got = splitTranscription(await ask(part, prompt), batch.start, batch);
} catch (err) {
// Configuration problems will not fix themselves on the next batch.
if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err;
if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
if (got) {
cache.batches[key] = got;
save();
}
}
}
if (got) sections.push(...got);
opts.onProgress?.(Math.min(limit, batch.end) / limit);
}
return { sections, warnings };
}
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
@ -279,15 +375,16 @@ export async function processDocument(app: AppContext, id: string, opts: { force
setProgress(0.82);
if (row.kind === 'image' || row.kind === 'pdf') {
try {
const transcribed = await transcribeWithVision(app, row, bytes);
const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) });
if (transcribed) warnings.push(...transcribed.warnings);
if (transcribed === null) {
warnings.push(
row.kind === 'image'
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
);
} else if (transcribed.length > 0) {
sections = transcribed;
} else if (transcribed.sections.length > 0) {
sections = transcribed.sections;
needsOcr = false;
} else {
warnings.push('The AI model did not find any readable text in this file.');
@ -325,11 +422,12 @@ export async function processDocument(app: AppContext, id: string, opts: { force
tokens,
chunks.length,
needsOcr ? 1 : 0,
JSON.stringify([...new Set(warnings)].slice(0, 8)),
JSON.stringify([...new Set(warnings)].slice(0, 12)),
now(),
id,
);
});
scheduleIndexing(app);
} catch (err) {
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
@ -432,7 +530,7 @@ export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' |
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
* so every part of the document is represented.
*/
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number; ranked?: ChunkHit[] }): ChunkHit[][] {
const { sql, params } = scopeSql(scope);
if (!sql) return [];
const all = db
@ -448,7 +546,8 @@ export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string;
let chosen = all;
if (opts.focus?.trim()) {
const ranked = searchChunks(db, opts.focus, scope, 400);
// Callers pass a hybrid (keyword + semantic) ranking when they have one.
const ranked = opts.ranked ?? searchChunks(db, opts.focus, scope, 400);
if (ranked.length >= 3) {
const keep = new Set<number>();
let used = 0;