Study/server/documents/extract.ts
Elijah 3e74ecd843 Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 08:16:41 -07:00

362 lines
15 KiB
TypeScript

/**
* Document text extraction: kind detection plus one extractor per format.
* Every failure surfaces as an ExtractionError with a user-facing message.
*/
import { cleanText } from './chunk.ts';
import { extractDocx } from './docx.ts';
import { extractPdf } from './pdf.ts';
import { extractPptx } from './pptx.ts';
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
import { listZip, ZipError, type ZipLimits } from './zip.ts';
/** Largest upload accepted (100 MB). */
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
export interface ExtractInput {
bytes: Uint8Array;
filename: string;
mime?: string;
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
onProgress?: (done: number, total: number) => void;
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
signal?: AbortSignal;
}
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
const SNIFF_BYTES = 256 * 1024;
function extensionOf(filename: string): string {
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
}
function baseMime(mime: string | undefined): string {
return (mime ?? '').split(';')[0].trim().toLowerCase();
}
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
if (bytes.length < offset + signature.length) return false;
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
return true;
}
function isImage(bytes: Uint8Array): boolean {
return (
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
);
}
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
function hasPdfHeader(bytes: Uint8Array): boolean {
const limit = Math.min(bytes.length - 5, 1024);
for (let i = 0; i <= limit; i++) {
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
}
return false;
}
function isZip(bytes: Uint8Array): boolean {
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
}
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
function isOle(bytes: Uint8Array): boolean {
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
}
function isUtf16Bom(bytes: Uint8Array): boolean {
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
}
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
try {
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
return true;
} catch {
return false;
}
}
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
function looksLikeSingleByteText(sample: Uint8Array): boolean {
let high = 0;
for (let i = 0; i < sample.length; i++) {
const b = sample[i];
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
if (b === 0x7f) return false;
if (b >= 0x80) high++;
}
return high <= sample.length * 0.3;
}
function isProbablyText(bytes: Uint8Array): boolean {
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
if (sample.includes(0)) return false;
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
}
/**
* Works out what a file is. Magic bytes win over the name; the extension and
* MIME type are only a fallback. Returns null for anything unsupported
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
*/
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
const ext = extensionOf(filename);
const type = baseMime(mime);
if (!bytes || bytes.length === 0) return null;
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
if (isImage(bytes)) return 'image';
if (isZip(bytes)) {
let names: string[] | null = null;
try {
names = listZip(bytes).map((entry) => entry.name);
} catch {
names = null;
}
if (names) {
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
if (names.includes('ppt/presentation.xml')) return 'pptx';
return null; // xlsx, odt, epub, jar, plain zip...
}
// Unreadable archive: trust the name so extraction can report it as damaged.
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
return null;
}
if (isOle(bytes)) {
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
if (ext === 'docx') return 'docx';
if (ext === 'pptx') return 'pptx';
return null;
}
// No recognisable signature: fall back to the name / declared type.
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
return null;
}
// ---------------------------------------------------------------------------
// Text and Markdown
// ---------------------------------------------------------------------------
function decodeText(bytes: Uint8Array, warnings: string[]): string {
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
return new TextDecoder('utf-8').decode(bytes.subarray(3));
}
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
try {
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch {
if (bytes.includes(0)) {
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
}
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
return new TextDecoder('windows-1252').decode(bytes);
}
}
/** Target size of a plain-text section, in characters. */
const TEXT_SECTION_CHARS = 6000;
function splitPlainText(text: string): ExtractedSection[] {
const sections: ExtractedSection[] = [];
let current: string[] = [];
let size = 0;
const flush = (separator: string) => {
if (current.length === 0) return;
sections.push({ text: current.join(separator) });
current = [];
size = 0;
};
for (const block of text.split(/\n{2,}/)) {
if (!block.trim()) continue;
if (block.length > TEXT_SECTION_CHARS) {
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
flush('\n\n');
for (const line of block.split('\n')) {
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
current.push(line);
size += line.length + 1;
}
flush('\n');
continue;
}
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
current.push(block);
size += block.length + 2;
}
flush('\n\n');
return sections;
}
function markdownHeadingText(raw: string): string {
return raw
.replace(/[ \t]+#+[ \t]*$/, '')
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\*\*|__|[*`]/g, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
}
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
const sections: ExtractedSection[] = [];
let title: string | undefined;
let heading: string | undefined;
let lines: string[] = [];
let fence: string | null = null;
const flush = () => {
const body = lines.join('\n').trim();
lines = [];
if (!body) return;
const section: ExtractedSection = { text: body };
if (heading) section.heading = heading;
sections.push(section);
};
for (const line of text.split('\n')) {
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
if (fenceMatch) {
const marker = fenceMatch[1][0];
if (fence === null) fence = marker;
else if (fence === marker) fence = null;
} else if (fence === null && line.length < 400) {
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
if (match) {
const headingText = markdownHeadingText(match[2]);
if (headingText) {
flush();
heading = headingText;
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
}
}
}
lines.push(line);
}
flush();
return { sections, title };
}
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
const warnings: string[] = [];
const text = cleanText(decodeText(bytes, warnings));
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
if (kind === 'markdown') {
const { sections, title } = splitMarkdown(text);
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
if (title) result.title = title;
return result;
}
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
const ext = extensionOf(filename);
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
}
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
}
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
}
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
if (error.code === 'too_large') {
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
}
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
}
/**
* Extracts the text of a document as ordered sections. Images return no
* sections and `needsOcr: true`; the caller runs vision transcription.
*/
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
const bytes = input?.bytes;
const filename = typeof input?.filename === 'string' ? input.filename : '';
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
}
if (bytes.length > MAX_DOCUMENT_BYTES) {
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
}
if (input.signal?.aborted) {
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
}
let kind: DocKind | null;
try {
kind = detectKind(filename, input.mime, bytes);
} catch (error) {
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
}
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
try {
switch (kind) {
case 'pdf':
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
case 'docx':
case 'pptx': {
if (isOle(bytes)) {
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
}
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
}
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
}
case 'text':
case 'markdown': {
const result = extractText(bytes, kind);
input.onProgress?.(1, 1);
return result;
}
case 'image':
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
}
} catch (error) {
if (error instanceof ExtractionError) throw error;
if (error instanceof ZipError) throw fromZipError(error, kind);
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
if (kind === 'docx' || kind === 'pptx') {
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
}
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
}
}