/** * Uploaded study materials: storage, the background processing queue * (extract -> optional AI transcription -> chunk -> index) and retrieval. */ import fs from 'node:fs'; import path from 'node:path'; import crypto from 'node:crypto'; import { resolveModel, runChat } from '../ai/service.ts'; import { AIError, type ContentPart } from '../ai/types.ts'; import type { DB } from '../db.ts'; import { chunkSections, estimateTokens } from '../documents/chunk.ts'; import { detectKind, extractDocument } from '../documents/extract.ts'; import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts'; import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts'; import { HttpError, notFound, type AppContext } from '../http.ts'; import { ftsQuery, newId, now, parseJson } from '../util.ts'; import { getClass, resolveFolder } from './content.ts'; import { scheduleIndexing } from './semantic.ts'; export interface DocumentRow { id: string; class_id: string; folder_id: string | null; title: string; filename: string; mime: string; kind: DocKind; size: number; sha256: string; storage_name: string; status: 'pending' | 'processing' | 'ready' | 'failed'; error: string; progress: number; page_count: number | null; char_count: number; token_count: number; chunk_count: number; needs_ocr: number; warnings: string; created_at: number; updated_at: number; } export const mapDocument = (r: DocumentRow) => ({ id: r.id, classId: r.class_id, folderId: r.folder_id, title: r.title, filename: r.filename, mime: r.mime, docKind: r.kind, size: r.size, status: r.status, error: r.error, progress: r.progress, pageCount: r.page_count, charCount: r.char_count, tokenCount: r.token_count, chunkCount: r.chunk_count, needsOcr: Boolean(r.needs_ocr), warnings: parseJson(r.warnings, []), createdAt: r.created_at, updatedAt: r.updated_at, }); export type DocumentDto = ReturnType; const MIME_BY_KIND: Record = { pdf: 'application/pdf', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', text: 'text/plain; charset=utf-8', markdown: 'text/markdown; charset=utf-8', image: 'application/octet-stream', }; function imageMime(bytes: Uint8Array): string { if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png'; if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg'; if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif'; if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp'; return 'application/octet-stream'; } export function getDocument(db: DB, id: string): DocumentRow { const row = db.get('SELECT * FROM documents WHERE id = ?', id); if (!row) throw notFound('Document'); return row; } export function documentPath(app: AppContext, row: Pick): string { // basename() guarantees a stored name can never escape the uploads folder. return path.join(app.config.uploadsDir, path.basename(row.storage_name)); } function titleFromFilename(filename: string): string { const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim(); return (base || 'Untitled document').slice(0, 200); } /* ------------------------------ ingest ------------------------------ */ export function ingestUpload( app: AppContext, input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string }, ): DocumentRow { const { db } = app; getClass(db, input.classId); if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`); if (input.bytes.length > app.config.maxUploadBytes) { throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`); } const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload'; let kind = detectKind(filename, input.mime, input.bytes); // A file whose name claims a binary format but whose bytes say otherwise (a renamed // executable, say) is refused here instead of being stored and failing later. if (kind === 'pdf' && !Buffer.from(input.bytes.subarray(0, 1024)).includes('%PDF')) kind = null; if ((kind === 'docx' || kind === 'pptx') && !(input.bytes[0] === 0x50 && input.bytes[1] === 0x4b) && !(input.bytes[0] === 0xd0 && input.bytes[1] === 0xcf)) kind = null; if (!kind) { throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`); } const id = newId(); const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind]; const storageName = `${id}${ext}`; fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes); const at = now(); const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1; db.run( `INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`, id, input.classId, resolveFolder(db, input.classId, input.folderId), (input.title?.trim() || titleFromFilename(filename)).slice(0, 200), filename, kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind], kind, input.bytes.length, crypto.createHash('sha256').update(input.bytes).digest('hex'), storageName, sort, at, at, ); enqueue(app, id); return getDocument(db, id); } export function deleteDocument(app: AppContext, id: string): void { const row = getDocument(app.db, id); app.db.tx(() => { app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id); app.db.run('DELETE FROM documents WHERE id = ?', id); }); fs.rmSync(documentPath(app, row), { force: true }); fs.rmSync(ocrCachePath(app, row), { force: true }); } /* ------------------------------ processing queue ------------------------------ */ const queues = new WeakMap; idle: Promise; wake: () => void }>(); function queueFor(app: AppContext) { let q = queues.get(app.db); if (!q) { q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined }; queues.set(app.db, q); } return q; } export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void { const q = queueFor(app); if (opts.ocr) q.ocr.add(id); if (!q.pending.includes(id)) q.pending.push(id); if (!q.running) { q.running = true; q.idle = (async () => { // Yield first so the HTTP response for the upload is sent before heavy parsing starts. await new Promise((r) => setTimeout(r, 10)); while (q.pending.length > 0) { const next = q.pending.shift()!; const forceOcr = q.ocr.delete(next); try { await processDocument(app, next, { forceOcr }); } catch (e) { app.log('error', `Document ${next} failed unexpectedly`, e); } } q.running = false; })(); } } /** Resolves when the queue is empty. Used by tests and graceful shutdown. */ export function queueIdle(app: AppContext): Promise { return queueFor(app).idle; } /** Re-queues anything a restart interrupted. */ export function resumeQueue(app: AppContext): number { const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`); for (const r of rows) enqueue(app, r.id); return rows.length; } const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown. - Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information. - Keep headings, lists and tables (as Markdown tables). - For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown. - For a multi-page document, start each page with a line of the form "## Page N". - If something is illegible, write [illegible]. Return only the transcription.`; /** Long scanned PDFs are read this many pages at a time. */ export const OCR_PAGES_PER_BATCH = 8; /** Largest single request sent to a model (providers cap inline files around 32 MB). */ const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024; /** Upper bound on pages read with AI for one document, to keep costs predictable. */ export const OCR_MAX_PAGES = 600; interface OcrCache { modelId: string; /** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */ batches: Record; } const ocrCachePath = (app: AppContext, row: Pick) => `${documentPath(app, row)}.ocr.json`; /** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */ export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] { const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im); if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : []; const sections: ExtractedSection[] = []; if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage }); for (let i = 1; i < pages.length; i += 2) { const body = (pages[i + 1] ?? '').trim(); let page = Number(pages[i]); // Models sometimes number a batch from 1; map those back onto the batch's real pages. if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1; if (body) sections.push({ text: body, page }); } return sections; } interface Transcription { sections: ExtractedSection[]; warnings: string[]; } /** * Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a * few pages, with finished batches cached beside the upload so a restart or a failed batch does * not repeat paid work. Returns null when no vision model is configured. */ async function transcribeWithVision( app: AppContext, row: DocumentRow, bytes: Uint8Array, opts: { fresh: boolean; onProgress?: (fraction: number) => void }, ): Promise { let vision; try { vision = resolveModel(app, 'vision'); } catch (err) { if (err instanceof AIError && err.code === 'not_configured') return null; throw err; } if (!vision.model.capabilities.vision) return null; const ask = async (part: ContentPart, prompt: string) => { const result = await runChat(app, { task: 'vision', purpose: 'document transcription', messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }], maxOutputTokens: 32000, reasoningEffort: 'low', }); return result.text.trim(); }; if (row.kind === 'image') { const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT); return { sections: text ? [{ text }] : [], warnings: [] }; } if (!vision.model.capabilities.pdf) { throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`); } const cacheFile = ocrCachePath(app, row); let cache: OcrCache = { modelId: vision.model.id, batches: {} }; if (opts.fresh) fs.rmSync(cacheFile, { force: true }); else { try { const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache; if (saved.modelId === vision.model.id && saved.batches) cache = saved; } catch { /* no cache yet */ } } const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache)); const warnings: string[] = []; let total: number; try { total = await pdfPageCount(bytes); } catch { throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.'); } const limit = Math.min(total, OCR_MAX_PAGES); if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`); const sections: ExtractedSection[] = []; for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) { if (batch.start > limit) break; const key = `${batch.start}-${batch.end}`; const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`; let got: ExtractedSection[] | undefined = cache.batches[key]; if (!got) { if (batch.bytes.length > OCR_MAX_BATCH_BYTES) { warnings.push(`${label} is too large to send to an AI model and was skipped.`); } else { const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`; const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` }; for (let attempt = 1; attempt <= 2 && !got; attempt++) { try { got = splitTranscription(await ask(part, prompt), batch.start, batch); } catch (err) { // Configuration problems will not fix themselves on the next batch. if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err; if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`); } } if (got) { cache.batches[key] = got; save(); } } } if (got) sections.push(...got); opts.onProgress?.(Math.min(limit, batch.end) / limit); } return { sections, warnings }; } export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise { const { db } = app; const row = db.get('SELECT * FROM documents WHERE id = ?', id); if (!row) return; const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id); db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id); try { const file = documentPath(app, row); if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.'); const bytes = new Uint8Array(await fs.promises.readFile(file)); let lastWrite = 0; const result = await extractDocument({ bytes, filename: row.filename, mime: row.mime, onProgress: (done, total) => { const t = Date.now(); if (t - lastWrite > 400 && total > 0) { lastWrite = t; setProgress((done / total) * 0.8); } }, }); const warnings = [...result.warnings]; let sections = result.sections; let needsOcr = result.needsOcr; if (needsOcr || opts.forceOcr) { setProgress(0.82); if (row.kind === 'image' || row.kind === 'pdf') { try { const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) }); if (transcribed) warnings.push(...transcribed.warnings); if (transcribed === null) { warnings.push( row.kind === 'image' ? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".' : 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".', ); } else if (transcribed.sections.length > 0) { sections = transcribed.sections; needsOcr = false; } else { warnings.push('The AI model did not find any readable text in this file.'); } } catch (err) { warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`); } } } const chunks = chunkSections(sections); const chars = sections.reduce((n, s) => n + s.text.length, 0); const tokens = chunks.reduce((n, c) => n + c.tokens, 0); db.tx(() => { db.run('DELETE FROM chunks WHERE document_id = ?', id); const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id); if (!current) return; // deleted while processing for (const c of chunks) { db.run( 'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', id, current.class_id, c.seq, c.text, c.heading, c.pageStart, c.pageEnd, c.tokens, ); } db.run( `UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`, result.pageCount ?? null, chars, tokens, chunks.length, needsOcr ? 1 : 0, JSON.stringify([...new Set(warnings)].slice(0, 12)), now(), id, ); }); scheduleIndexing(app); } catch (err) { const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.'; if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err); db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id); } } /* ------------------------------ retrieval ------------------------------ */ export interface ChunkHit { id: number; documentId: string; documentTitle: string; seq: number; text: string; heading: string; pageStart: number | null; pageEnd: number | null; tokens: number; score?: number; } type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number }; const mapHit = (r: ChunkJoin): ChunkHit => ({ id: r.id, documentId: r.document_id, documentTitle: r.doc_title, seq: r.seq, text: r.text, heading: r.heading, pageStart: r.page_start, pageEnd: r.page_end, tokens: r.tokens, score: r.score, }); export interface ChunkScope { classId?: string | null; documentIds?: string[]; } function scopeSql(scope: ChunkScope): { sql: string; params: string[] } { if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds }; if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] }; return { sql: '', params: [] }; } /** BM25 full-text search over document chunks. */ export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] { const match = ftsQuery(query, 'or'); if (!match) return []; const { sql, params } = scopeSql(scope); try { return db .all( `SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`, match, ...params, limit, ) .map(mapHit); } catch { return []; } } export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] { return db .all( `SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`, documentId, offset, limit, ) .map(mapHit); } export function chunkById(db: DB, id: number): ChunkHit | null { const row = db.get( `SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`, id, ); return row ? mapHit(row) : null; } export function sourceLabel(hit: Pick): string { if (hit.pageStart === null) return hit.documentTitle; return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`; } /** * Chooses source material for a generation job and splits it into `groups` batches that * each fit `budgetTokens`. * - With a focus, the best-matching chunks are used (kept in document order). * - Without one, the whole material is used; if it is too long, chunks are sampled evenly * so every part of the document is represented. */ export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number; ranked?: ChunkHit[] }): ChunkHit[][] { const { sql, params } = scopeSql(scope); if (!sql) return []; const all = db .all( `SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`, ...params, ) .map(mapHit); if (all.length === 0) return []; const capacity = opts.budgetTokens * opts.groups; const total = all.reduce((n, c) => n + c.tokens, 0); let chosen = all; if (opts.focus?.trim()) { // Callers pass a hybrid (keyword + semantic) ranking when they have one. const ranked = opts.ranked ?? searchChunks(db, opts.focus, scope, 400); if (ranked.length >= 3) { const keep = new Set(); let used = 0; for (const hit of ranked) { if (used + hit.tokens > capacity) break; keep.add(hit.id); used += hit.tokens; } chosen = all.filter((c) => keep.has(c.id)); } } if (chosen === all && total > capacity) { // Even sampling: walk the document and keep chunks at a steady stride. const stride = total / capacity; const picked: ChunkHit[] = []; let acc = 0; let next = 0; for (const c of all) { if (acc >= next) { picked.push(c); next += c.tokens * stride; } acc += c.tokens; } chosen = picked; } // Contiguous groups of roughly equal size, none above the budget. const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0); const groupCount = Math.max(1, Math.min(opts.groups, chosen.length)); const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount)); const groups: ChunkHit[][] = []; let current: ChunkHit[] = []; let size = 0; for (const c of chosen) { if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) { groups.push(current); current = []; size = 0; } if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full current.push(c); size += c.tokens; } if (current.length > 0) groups.push(current); return groups; } export function formatExcerpts(chunks: ChunkHit[]): string { return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n'); } export { estimateTokens };