Add drag reordering, exam deadlines, frozen shared quizzes, batched OCR and semantic search
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
- Drag-to-reorder for class materials, folders, quiz questions and deck cards. Pointer-based so touch works, plus keyboard (arrows/Home/End) with screen-reader announcements. - Timed exams get a wall-clock deadline. The server refuses late answers, submits overdue exams with what was saved, and the countdown keeps running while the page is closed. - Shared quiz sessions keep a frozen copy of their questions and flag later owner edits. - Scanned PDFs are read in cached 8-page batches with real page numbers (pdf-lib), removing the 30 MB whole-file limit. Compatible providers now receive PDFs as file parts. - Semantic search: embeddings for OpenAI, Gemini, Ollama and compatible servers, background indexing into chunk_vectors (migration 2), hybrid keyword + vector ranking for search, the assistant and focused generation, and a Settings -> AI section. 181 tests pass locally and in the Docker test stage on Node 24. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3046e46d1c
commit
ccdad9728b
39 changed files with 1772 additions and 121 deletions
|
|
@ -10,10 +10,12 @@ import { AIError, type ContentPart } from '../ai/types.ts';
|
|||
import type { DB } from '../db.ts';
|
||||
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
|
||||
import { detectKind, extractDocument } from '../documents/extract.ts';
|
||||
import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts';
|
||||
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
|
||||
import { HttpError, notFound, type AppContext } from '../http.ts';
|
||||
import { ftsQuery, newId, now, parseJson } from '../util.ts';
|
||||
import { getClass, resolveFolder } from './content.ts';
|
||||
import { scheduleIndexing } from './semantic.ts';
|
||||
|
||||
export interface DocumentRow {
|
||||
id: string;
|
||||
|
|
@ -150,6 +152,7 @@ export function deleteDocument(app: AppContext, id: string): void {
|
|||
app.db.run('DELETE FROM documents WHERE id = ?', id);
|
||||
});
|
||||
fs.rmSync(documentPath(app, row), { force: true });
|
||||
fs.rmSync(ocrCachePath(app, row), { force: true });
|
||||
}
|
||||
|
||||
/* ------------------------------ processing queue ------------------------------ */
|
||||
|
|
@ -208,43 +211,136 @@ const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
|
|||
- If something is illegible, write [illegible].
|
||||
Return only the transcription.`;
|
||||
|
||||
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
|
||||
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
|
||||
const data = Buffer.from(bytes).toString('base64');
|
||||
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
|
||||
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
|
||||
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
|
||||
/** Long scanned PDFs are read this many pages at a time. */
|
||||
export const OCR_PAGES_PER_BATCH = 8;
|
||||
/** Largest single request sent to a model (providers cap inline files around 32 MB). */
|
||||
const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024;
|
||||
/** Upper bound on pages read with AI for one document, to keep costs predictable. */
|
||||
export const OCR_MAX_PAGES = 600;
|
||||
|
||||
interface OcrCache {
|
||||
modelId: string;
|
||||
/** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */
|
||||
batches: Record<string, ExtractedSection[]>;
|
||||
}
|
||||
|
||||
const ocrCachePath = (app: AppContext, row: Pick<DocumentRow, 'storage_name'>) => `${documentPath(app, row)}.ocr.json`;
|
||||
|
||||
/** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */
|
||||
export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] {
|
||||
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
|
||||
if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : [];
|
||||
const sections: ExtractedSection[] = [];
|
||||
if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage });
|
||||
for (let i = 1; i < pages.length; i += 2) {
|
||||
const body = (pages[i + 1] ?? '').trim();
|
||||
let page = Number(pages[i]);
|
||||
// Models sometimes number a batch from 1; map those back onto the batch's real pages.
|
||||
if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1;
|
||||
if (body) sections.push({ text: body, page });
|
||||
}
|
||||
let result;
|
||||
return sections;
|
||||
}
|
||||
|
||||
interface Transcription {
|
||||
sections: ExtractedSection[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a
|
||||
* few pages, with finished batches cached beside the upload so a restart or a failed batch does
|
||||
* not repeat paid work. Returns null when no vision model is configured.
|
||||
*/
|
||||
async function transcribeWithVision(
|
||||
app: AppContext,
|
||||
row: DocumentRow,
|
||||
bytes: Uint8Array,
|
||||
opts: { fresh: boolean; onProgress?: (fraction: number) => void },
|
||||
): Promise<Transcription | null> {
|
||||
let vision;
|
||||
try {
|
||||
const vision = resolveModel(app, 'vision');
|
||||
if (!vision.model.capabilities.vision) return null;
|
||||
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
|
||||
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
|
||||
}
|
||||
result = await runChat(app, {
|
||||
task: 'vision',
|
||||
purpose: 'document transcription',
|
||||
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
|
||||
maxOutputTokens: 32000,
|
||||
reasoningEffort: 'low',
|
||||
});
|
||||
vision = resolveModel(app, 'vision');
|
||||
} catch (err) {
|
||||
if (err instanceof AIError && err.code === 'not_configured') return null;
|
||||
throw err;
|
||||
}
|
||||
const text = result.text.trim();
|
||||
if (!text) return [];
|
||||
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
|
||||
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
|
||||
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
|
||||
const sections: ExtractedSection[] = [];
|
||||
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
|
||||
for (let i = 1; i < pages.length; i += 2) {
|
||||
const body = (pages[i + 1] ?? '').trim();
|
||||
if (body) sections.push({ text: body, page: Number(pages[i]) });
|
||||
if (!vision.model.capabilities.vision) return null;
|
||||
|
||||
const ask = async (part: ContentPart, prompt: string) => {
|
||||
const result = await runChat(app, {
|
||||
task: 'vision',
|
||||
purpose: 'document transcription',
|
||||
messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }],
|
||||
maxOutputTokens: 32000,
|
||||
reasoningEffort: 'low',
|
||||
});
|
||||
return result.text.trim();
|
||||
};
|
||||
|
||||
if (row.kind === 'image') {
|
||||
const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT);
|
||||
return { sections: text ? [{ text }] : [], warnings: [] };
|
||||
}
|
||||
return sections;
|
||||
|
||||
if (!vision.model.capabilities.pdf) {
|
||||
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
|
||||
}
|
||||
|
||||
const cacheFile = ocrCachePath(app, row);
|
||||
let cache: OcrCache = { modelId: vision.model.id, batches: {} };
|
||||
if (opts.fresh) fs.rmSync(cacheFile, { force: true });
|
||||
else {
|
||||
try {
|
||||
const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache;
|
||||
if (saved.modelId === vision.model.id && saved.batches) cache = saved;
|
||||
} catch {
|
||||
/* no cache yet */
|
||||
}
|
||||
}
|
||||
const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache));
|
||||
|
||||
const warnings: string[] = [];
|
||||
let total: number;
|
||||
try {
|
||||
total = await pdfPageCount(bytes);
|
||||
} catch {
|
||||
throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.');
|
||||
}
|
||||
const limit = Math.min(total, OCR_MAX_PAGES);
|
||||
if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`);
|
||||
|
||||
const sections: ExtractedSection[] = [];
|
||||
for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) {
|
||||
if (batch.start > limit) break;
|
||||
const key = `${batch.start}-${batch.end}`;
|
||||
const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`;
|
||||
let got: ExtractedSection[] | undefined = cache.batches[key];
|
||||
if (!got) {
|
||||
if (batch.bytes.length > OCR_MAX_BATCH_BYTES) {
|
||||
warnings.push(`${label} is too large to send to an AI model and was skipped.`);
|
||||
} else {
|
||||
const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`;
|
||||
const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` };
|
||||
for (let attempt = 1; attempt <= 2 && !got; attempt++) {
|
||||
try {
|
||||
got = splitTranscription(await ask(part, prompt), batch.start, batch);
|
||||
} catch (err) {
|
||||
// Configuration problems will not fix themselves on the next batch.
|
||||
if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err;
|
||||
if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`);
|
||||
}
|
||||
}
|
||||
if (got) {
|
||||
cache.batches[key] = got;
|
||||
save();
|
||||
}
|
||||
}
|
||||
}
|
||||
if (got) sections.push(...got);
|
||||
opts.onProgress?.(Math.min(limit, batch.end) / limit);
|
||||
}
|
||||
return { sections, warnings };
|
||||
}
|
||||
|
||||
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
|
||||
|
|
@ -279,15 +375,16 @@ export async function processDocument(app: AppContext, id: string, opts: { force
|
|||
setProgress(0.82);
|
||||
if (row.kind === 'image' || row.kind === 'pdf') {
|
||||
try {
|
||||
const transcribed = await transcribeWithVision(app, row, bytes);
|
||||
const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) });
|
||||
if (transcribed) warnings.push(...transcribed.warnings);
|
||||
if (transcribed === null) {
|
||||
warnings.push(
|
||||
row.kind === 'image'
|
||||
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
|
||||
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
|
||||
);
|
||||
} else if (transcribed.length > 0) {
|
||||
sections = transcribed;
|
||||
} else if (transcribed.sections.length > 0) {
|
||||
sections = transcribed.sections;
|
||||
needsOcr = false;
|
||||
} else {
|
||||
warnings.push('The AI model did not find any readable text in this file.');
|
||||
|
|
@ -325,11 +422,12 @@ export async function processDocument(app: AppContext, id: string, opts: { force
|
|||
tokens,
|
||||
chunks.length,
|
||||
needsOcr ? 1 : 0,
|
||||
JSON.stringify([...new Set(warnings)].slice(0, 8)),
|
||||
JSON.stringify([...new Set(warnings)].slice(0, 12)),
|
||||
now(),
|
||||
id,
|
||||
);
|
||||
});
|
||||
scheduleIndexing(app);
|
||||
} catch (err) {
|
||||
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
|
||||
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
|
||||
|
|
@ -432,7 +530,7 @@ export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' |
|
|||
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
|
||||
* so every part of the document is represented.
|
||||
*/
|
||||
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
|
||||
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number; ranked?: ChunkHit[] }): ChunkHit[][] {
|
||||
const { sql, params } = scopeSql(scope);
|
||||
if (!sql) return [];
|
||||
const all = db
|
||||
|
|
@ -448,7 +546,8 @@ export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string;
|
|||
let chosen = all;
|
||||
|
||||
if (opts.focus?.trim()) {
|
||||
const ranked = searchChunks(db, opts.focus, scope, 400);
|
||||
// Callers pass a hybrid (keyword + semantic) ranking when they have one.
|
||||
const ranked = opts.ranked ?? searchChunks(db, opts.focus, scope, 400);
|
||||
if (ranked.length >= 3) {
|
||||
const keep = new Set<number>();
|
||||
let used = 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue