/** * Document text extraction: kind detection plus one extractor per format. * Every failure surfaces as an ExtractionError with a user-facing message. */ import { cleanText } from './chunk.ts'; import { extractDocx } from './docx.ts'; import { extractPdf } from './pdf.ts'; import { extractPptx } from './pptx.ts'; import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts'; import { listZip, ZipError, type ZipLimits } from './zip.ts'; /** Largest upload accepted (100 MB). */ export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024; /** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */ const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 }; export interface ExtractInput { bytes: Uint8Array; filename: string; mime?: string; /** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */ onProgress?: (done: number, total: number) => void; /** Aborting rejects with ExtractionError('failed') at the next page boundary. */ signal?: AbortSignal; } const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']); const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']); const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']); const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']); const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'; const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'; const SNIFF_BYTES = 256 * 1024; function extensionOf(filename: string): string { const base = (filename ?? '').split(/[\\/]/).pop() ?? ''; const dot = base.lastIndexOf('.'); return dot > 0 ? base.slice(dot + 1).toLowerCase() : ''; } function baseMime(mime: string | undefined): string { return (mime ?? '').split(';')[0].trim().toLowerCase(); } function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean { if (bytes.length < offset + signature.length) return false; for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false; return true; } function isImage(bytes: Uint8Array): boolean { return ( startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP ); } /** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */ function hasPdfHeader(bytes: Uint8Array): boolean { const limit = Math.min(bytes.length - 5, 1024); for (let i = 0; i <= limit; i++) { if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true; } return false; } function isZip(bytes: Uint8Array): boolean { return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]); } /** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */ function isOle(bytes: Uint8Array): boolean { return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]); } function isUtf16Bom(bytes: Uint8Array): boolean { return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff)); } function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean { try { new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail }); return true; } catch { return false; } } /** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */ function looksLikeSingleByteText(sample: Uint8Array): boolean { let high = 0; for (let i = 0; i < sample.length; i++) { const b = sample[i]; if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false; if (b === 0x7f) return false; if (b >= 0x80) high++; } return high <= sample.length * 0.3; } function isProbablyText(bytes: Uint8Array): boolean { if (isUtf16Bom(bytes)) return bytes.length % 2 === 0; const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes; if (sample.includes(0)) return false; return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample); } /** * Works out what a file is. Magic bytes win over the name; the extension and * MIME type are only a fallback. Returns null for anything unsupported * (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk). */ export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null { const ext = extensionOf(filename); const type = baseMime(mime); if (!bytes || bytes.length === 0) return null; if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf'; if (isImage(bytes)) return 'image'; if (isZip(bytes)) { let names: string[] | null = null; try { names = listZip(bytes).map((entry) => entry.name); } catch { names = null; } if (names) { if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx'; if (names.includes('ppt/presentation.xml')) return 'pptx'; return null; // xlsx, odt, epub, jar, plain zip... } // Unreadable archive: trust the name so extraction can report it as damaged. if (ext === 'docx' || type === DOCX_MIME) return 'docx'; if (ext === 'pptx' || type === PPTX_MIME) return 'pptx'; return null; } if (isOle(bytes)) { // Password-protected .docx/.pptx are OLE containers; extraction explains that. if (ext === 'docx') return 'docx'; if (ext === 'pptx') return 'pptx'; return null; } // No recognisable signature: fall back to the name / declared type. if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf'; if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx'; if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx'; if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null; if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null; return null; } // --------------------------------------------------------------------------- // Text and Markdown // --------------------------------------------------------------------------- function decodeText(bytes: Uint8Array, warnings: string[]): string { if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { return new TextDecoder('utf-8').decode(bytes.subarray(3)); } if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2)); if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2)); try { return new TextDecoder('utf-8', { fatal: true }).decode(bytes); } catch { if (bytes.includes(0)) { throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file."); } warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.'); return new TextDecoder('windows-1252').decode(bytes); } } /** Target size of a plain-text section, in characters. */ const TEXT_SECTION_CHARS = 6000; function splitPlainText(text: string): ExtractedSection[] { const sections: ExtractedSection[] = []; let current: string[] = []; let size = 0; const flush = (separator: string) => { if (current.length === 0) return; sections.push({ text: current.join(separator) }); current = []; size = 0; }; for (const block of text.split(/\n{2,}/)) { if (!block.trim()) continue; if (block.length > TEXT_SECTION_CHARS) { // One huge block (CSV, logs, text without blank lines): fall back to line boundaries. flush('\n\n'); for (const line of block.split('\n')) { if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n'); current.push(line); size += line.length + 1; } flush('\n'); continue; } if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n'); current.push(block); size += block.length + 2; } flush('\n\n'); return sections; } function markdownHeadingText(raw: string): string { return raw .replace(/[ \t]+#+[ \t]*$/, '') .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') .replace(/\*\*|__|[*`]/g, '') .replace(/\s+/g, ' ') .trim() .slice(0, 200); } function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } { const sections: ExtractedSection[] = []; let title: string | undefined; let heading: string | undefined; let lines: string[] = []; let fence: string | null = null; const flush = () => { const body = lines.join('\n').trim(); lines = []; if (!body) return; const section: ExtractedSection = { text: body }; if (heading) section.heading = heading; sections.push(section); }; for (const line of text.split('\n')) { const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null; if (fenceMatch) { const marker = fenceMatch[1][0]; if (fence === null) fence = marker; else if (fence === marker) fence = null; } else if (fence === null && line.length < 400) { const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line); if (match) { const headingText = markdownHeadingText(match[2]); if (headingText) { flush(); heading = headingText; if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText; } } } lines.push(line); } flush(); return { sections, title }; } function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult { const warnings: string[] = []; const text = cleanText(decodeText(bytes, warnings)); if (!text) throw new ExtractionError('empty', "This file doesn't contain any text."); if (kind === 'markdown') { const { sections, title } = splitMarkdown(text); const result: ExtractResult = { kind, sections, warnings, needsOcr: false }; if (title) result.title = title; return result; } return { kind, sections: splitPlainText(text), warnings, needsOcr: false }; } // --------------------------------------------------------------------------- // Entry point // --------------------------------------------------------------------------- const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le'); function unsupportedMessage(filename: string, bytes: Uint8Array): string { const ext = extensionOf(filename); if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead."; if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead."; if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead."; if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') { return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`; } if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) { return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file."; } if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead."; return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file."; } function fromZipError(error: ZipError, kind: DocKind): ExtractionError { const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document'; if (error.code === 'too_large') { return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error }); } return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error }); } /** * Extracts the text of a document as ordered sections. Images return no * sections and `needsOcr: true`; the caller runs vision transcription. */ export async function extractDocument(input: ExtractInput): Promise { const bytes = input?.bytes; const filename = typeof input?.filename === 'string' ? input.filename : ''; if (!(bytes instanceof Uint8Array) || bytes.length === 0) { throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.'); } if (bytes.length > MAX_DOCUMENT_BYTES) { throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`); } if (input.signal?.aborted) { throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason }); } let kind: DocKind | null; try { kind = detectKind(filename, input.mime, bytes); } catch (error) { throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error }); } if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes)); try { switch (kind) { case 'pdf': return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal }); case 'docx': case 'pptx': { if (isOle(bytes)) { const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) { throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`); } throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes)); } const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal }; return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx); } case 'text': case 'markdown': { const result = extractText(bytes, kind); input.onProgress?.(1, 1); return result; } case 'image': return { kind: 'image', sections: [], needsOcr: true, warnings: [] }; } } catch (error) { if (error instanceof ExtractionError) throw error; if (error instanceof ZipError) throw fromZipError(error, kind); const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file'; if (kind === 'docx' || kind === 'pptx') { throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error }); } throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error }); } }