V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
362 lines
15 KiB
TypeScript
362 lines
15 KiB
TypeScript
/**
|
|
* Document text extraction: kind detection plus one extractor per format.
|
|
* Every failure surfaces as an ExtractionError with a user-facing message.
|
|
*/
|
|
import { cleanText } from './chunk.ts';
|
|
import { extractDocx } from './docx.ts';
|
|
import { extractPdf } from './pdf.ts';
|
|
import { extractPptx } from './pptx.ts';
|
|
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
import { listZip, ZipError, type ZipLimits } from './zip.ts';
|
|
|
|
/** Largest upload accepted (100 MB). */
|
|
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
|
|
|
|
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
|
|
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
|
|
|
|
export interface ExtractInput {
|
|
bytes: Uint8Array;
|
|
filename: string;
|
|
mime?: string;
|
|
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
|
|
onProgress?: (done: number, total: number) => void;
|
|
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
|
|
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
|
|
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
|
|
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
|
|
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
|
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
|
|
const SNIFF_BYTES = 256 * 1024;
|
|
|
|
function extensionOf(filename: string): string {
|
|
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
|
|
const dot = base.lastIndexOf('.');
|
|
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
|
|
}
|
|
|
|
function baseMime(mime: string | undefined): string {
|
|
return (mime ?? '').split(';')[0].trim().toLowerCase();
|
|
}
|
|
|
|
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
|
|
if (bytes.length < offset + signature.length) return false;
|
|
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
|
|
return true;
|
|
}
|
|
|
|
function isImage(bytes: Uint8Array): boolean {
|
|
return (
|
|
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
|
|
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
|
|
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
|
|
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
|
|
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
|
|
);
|
|
}
|
|
|
|
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
|
|
function hasPdfHeader(bytes: Uint8Array): boolean {
|
|
const limit = Math.min(bytes.length - 5, 1024);
|
|
for (let i = 0; i <= limit; i++) {
|
|
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function isZip(bytes: Uint8Array): boolean {
|
|
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
|
|
}
|
|
|
|
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
|
|
function isOle(bytes: Uint8Array): boolean {
|
|
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
|
}
|
|
|
|
function isUtf16Bom(bytes: Uint8Array): boolean {
|
|
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
|
|
}
|
|
|
|
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
|
|
try {
|
|
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
|
|
function looksLikeSingleByteText(sample: Uint8Array): boolean {
|
|
let high = 0;
|
|
for (let i = 0; i < sample.length; i++) {
|
|
const b = sample[i];
|
|
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
|
|
if (b === 0x7f) return false;
|
|
if (b >= 0x80) high++;
|
|
}
|
|
return high <= sample.length * 0.3;
|
|
}
|
|
|
|
function isProbablyText(bytes: Uint8Array): boolean {
|
|
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
|
|
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
|
|
if (sample.includes(0)) return false;
|
|
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
|
|
}
|
|
|
|
/**
|
|
* Works out what a file is. Magic bytes win over the name; the extension and
|
|
* MIME type are only a fallback. Returns null for anything unsupported
|
|
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
|
|
*/
|
|
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
|
|
const ext = extensionOf(filename);
|
|
const type = baseMime(mime);
|
|
if (!bytes || bytes.length === 0) return null;
|
|
|
|
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
|
|
if (isImage(bytes)) return 'image';
|
|
|
|
if (isZip(bytes)) {
|
|
let names: string[] | null = null;
|
|
try {
|
|
names = listZip(bytes).map((entry) => entry.name);
|
|
} catch {
|
|
names = null;
|
|
}
|
|
if (names) {
|
|
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
|
|
if (names.includes('ppt/presentation.xml')) return 'pptx';
|
|
return null; // xlsx, odt, epub, jar, plain zip...
|
|
}
|
|
// Unreadable archive: trust the name so extraction can report it as damaged.
|
|
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
|
|
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
|
|
return null;
|
|
}
|
|
|
|
if (isOle(bytes)) {
|
|
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
|
|
if (ext === 'docx') return 'docx';
|
|
if (ext === 'pptx') return 'pptx';
|
|
return null;
|
|
}
|
|
|
|
// No recognisable signature: fall back to the name / declared type.
|
|
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
|
|
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
|
|
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
|
|
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
|
|
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
|
|
return null;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Text and Markdown
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function decodeText(bytes: Uint8Array, warnings: string[]): string {
|
|
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
|
return new TextDecoder('utf-8').decode(bytes.subarray(3));
|
|
}
|
|
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
|
|
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
|
|
try {
|
|
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
|
} catch {
|
|
if (bytes.includes(0)) {
|
|
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
|
|
}
|
|
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
|
|
return new TextDecoder('windows-1252').decode(bytes);
|
|
}
|
|
}
|
|
|
|
/** Target size of a plain-text section, in characters. */
|
|
const TEXT_SECTION_CHARS = 6000;
|
|
|
|
function splitPlainText(text: string): ExtractedSection[] {
|
|
const sections: ExtractedSection[] = [];
|
|
let current: string[] = [];
|
|
let size = 0;
|
|
const flush = (separator: string) => {
|
|
if (current.length === 0) return;
|
|
sections.push({ text: current.join(separator) });
|
|
current = [];
|
|
size = 0;
|
|
};
|
|
for (const block of text.split(/\n{2,}/)) {
|
|
if (!block.trim()) continue;
|
|
if (block.length > TEXT_SECTION_CHARS) {
|
|
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
|
|
flush('\n\n');
|
|
for (const line of block.split('\n')) {
|
|
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
|
|
current.push(line);
|
|
size += line.length + 1;
|
|
}
|
|
flush('\n');
|
|
continue;
|
|
}
|
|
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
|
|
current.push(block);
|
|
size += block.length + 2;
|
|
}
|
|
flush('\n\n');
|
|
return sections;
|
|
}
|
|
|
|
function markdownHeadingText(raw: string): string {
|
|
return raw
|
|
.replace(/[ \t]+#+[ \t]*$/, '')
|
|
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
|
|
.replace(/\*\*|__|[*`]/g, '')
|
|
.replace(/\s+/g, ' ')
|
|
.trim()
|
|
.slice(0, 200);
|
|
}
|
|
|
|
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
|
|
const sections: ExtractedSection[] = [];
|
|
let title: string | undefined;
|
|
let heading: string | undefined;
|
|
let lines: string[] = [];
|
|
let fence: string | null = null;
|
|
const flush = () => {
|
|
const body = lines.join('\n').trim();
|
|
lines = [];
|
|
if (!body) return;
|
|
const section: ExtractedSection = { text: body };
|
|
if (heading) section.heading = heading;
|
|
sections.push(section);
|
|
};
|
|
for (const line of text.split('\n')) {
|
|
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
|
|
if (fenceMatch) {
|
|
const marker = fenceMatch[1][0];
|
|
if (fence === null) fence = marker;
|
|
else if (fence === marker) fence = null;
|
|
} else if (fence === null && line.length < 400) {
|
|
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
|
|
if (match) {
|
|
const headingText = markdownHeadingText(match[2]);
|
|
if (headingText) {
|
|
flush();
|
|
heading = headingText;
|
|
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
|
|
}
|
|
}
|
|
}
|
|
lines.push(line);
|
|
}
|
|
flush();
|
|
return { sections, title };
|
|
}
|
|
|
|
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
|
|
const warnings: string[] = [];
|
|
const text = cleanText(decodeText(bytes, warnings));
|
|
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
|
|
if (kind === 'markdown') {
|
|
const { sections, title } = splitMarkdown(text);
|
|
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
|
|
if (title) result.title = title;
|
|
return result;
|
|
}
|
|
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Entry point
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
|
|
|
|
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
|
|
const ext = extensionOf(filename);
|
|
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
|
|
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
|
|
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
|
|
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
|
|
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
|
|
}
|
|
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
|
|
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
|
|
}
|
|
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
|
|
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
|
|
}
|
|
|
|
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
|
|
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
|
|
if (error.code === 'too_large') {
|
|
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
|
|
}
|
|
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
|
|
}
|
|
|
|
/**
|
|
* Extracts the text of a document as ordered sections. Images return no
|
|
* sections and `needsOcr: true`; the caller runs vision transcription.
|
|
*/
|
|
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
|
|
const bytes = input?.bytes;
|
|
const filename = typeof input?.filename === 'string' ? input.filename : '';
|
|
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
|
|
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
|
|
}
|
|
if (bytes.length > MAX_DOCUMENT_BYTES) {
|
|
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
|
|
}
|
|
if (input.signal?.aborted) {
|
|
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
|
|
}
|
|
|
|
let kind: DocKind | null;
|
|
try {
|
|
kind = detectKind(filename, input.mime, bytes);
|
|
} catch (error) {
|
|
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
|
|
}
|
|
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
|
|
|
|
try {
|
|
switch (kind) {
|
|
case 'pdf':
|
|
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
|
|
case 'docx':
|
|
case 'pptx': {
|
|
if (isOle(bytes)) {
|
|
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
|
|
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
|
|
}
|
|
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
|
|
}
|
|
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
|
|
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
|
|
}
|
|
case 'text':
|
|
case 'markdown': {
|
|
const result = extractText(bytes, kind);
|
|
input.onProgress?.(1, 1);
|
|
return result;
|
|
}
|
|
case 'image':
|
|
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof ExtractionError) throw error;
|
|
if (error instanceof ZipError) throw fromZipError(error, kind);
|
|
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
|
|
if (kind === 'docx' || kind === 'pptx') {
|
|
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
|
|
}
|
|
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
|
|
}
|
|
}
|