/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */ import { getDocumentProxy } from 'unpdf'; import { cleanText, stripRepeatedPageLines } from './chunk.ts'; import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts'; export interface PdfContext { onProgress?: (done: number, total: number) => void; signal?: AbortSignal; } /** Below this average of characters per page the PDF is treated as a scan. */ const OCR_CHARS_PER_PAGE = 25; /** Release pdf.js caches (fonts, parsed objects) every this many pages. */ const CLEANUP_EVERY = 40; interface TextItemLike { str?: unknown; hasEOL?: unknown; transform?: unknown; height?: unknown; } interface Line { text: string; y: number | null; height: number; } /** * Rebuilds page text from pdf.js text items. Items arrive in content-stream * order (which keeps multi-column pages readable) with `hasEOL` marking line * ends; the vertical gaps between lines are then used to restore paragraph * breaks, which pdf.js does not report. */ export function textItemsToText(items: readonly TextItemLike[]): string { const lines: Line[] = []; let text = ''; let y: number | null = null; let height = 0; const endLine = () => { lines.push({ text, y, height }); text = ''; y = null; height = 0; }; for (const item of items) { if (!item || typeof item.str !== 'string') continue; const str = item.str; if (str.length > 0) { if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) { const t = item.transform as number[]; const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3; if (upright && Number.isFinite(t[5])) { y = t[5]; const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0); height = Number.isFinite(h) ? h : 0; } } text += str; } if (item.hasEOL === true) endLine(); } if (text.length > 0) endLine(); // Typical line advance on this page (median of the downward steps). const advances: number[] = []; for (let i = 1; i < lines.length; i++) { const a = lines[i - 1]; const b = lines[i]; if (a.y === null || b.y === null) continue; const dy = a.y - b.y; const ref = Math.max(a.height, b.height, 1); if (dy > 0 && dy < ref * 4) advances.push(dy); } advances.sort((p, q) => p - q); const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0; let out = ''; let prev: Line | null = null; for (const line of lines) { const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim(); if (!content) { // An empty line is an explicit gap. if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n'; continue; } if (out && !out.endsWith('\n')) { let gap = false; if (prev && prev.y !== null && line.y !== null) { const dy = prev.y - line.y; const ref = Math.max(prev.height, line.height, 1); if (dy < -ref * 2) gap = true; // jumped back up: new column or block else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3; else gap = dy > ref * 1.9; } out += gap ? '\n\n' : '\n'; } out += content; prev = line; } return out; } const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i; export function tidyTitle(raw: unknown): string | undefined { if (typeof raw !== 'string') return undefined; let title = cleanText(raw).replace(/\s+/g, ' ').trim(); title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim(); if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined; return title; } function errorName(error: unknown): string { return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : ''; } function toExtractionError(error: unknown): ExtractionError { if (error instanceof ExtractionError) return error; const name = errorName(error); const message = error instanceof Error ? error.message : String(error); if (name === 'PasswordException' || /password/i.test(message)) { return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error }); } if ( name === 'InvalidPDFException' || name === 'FormatError' || name === 'XRefParseException' || name === 'XRefEntryException' || name === 'MissingDataException' || /invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message) ) { return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error }); } return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error }); } function cancelled(signal: AbortSignal): ExtractionError { return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason }); } export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise { const { onProgress, signal } = ctx; if (signal?.aborted) throw cancelled(signal); let pdf: Awaited>; try { // pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy. const options: Record = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false }; pdf = await getDocumentProxy(bytes.slice(), options); } catch (error) { throw toExtractionError(error); } try { const total = pdf.numPages; if (!Number.isFinite(total) || total < 1) { throw new ExtractionError('corrupt', "This PDF doesn't contain any pages."); } let title: string | undefined; try { const meta = await pdf.getMetadata(); const info = (meta?.info ?? {}) as Record; title = tidyTitle(info.Title); if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title')); } catch { // Metadata is optional. } const pageTexts: string[] = new Array(total).fill(''); let failedPages = 0; let lastYield = Date.now(); for (let n = 1; n <= total; n++) { if (signal?.aborted) throw cancelled(signal); try { const page = await pdf.getPage(n); try { const content = await page.getTextContent(); pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]); } finally { page.cleanup(); } } catch (error) { if (errorName(error) === 'PasswordException') throw error; failedPages++; } if (n % CLEANUP_EVERY === 0) { try { await pdf.cleanup(); } catch { // Best effort. } } onProgress?.(n, total); // pdf.js runs on this thread in Node: give the event loop a turn regularly. if (Date.now() - lastYield > 20) { await new Promise((resolve) => setImmediate(resolve)); lastYield = Date.now(); } } if (failedPages === total) { throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.'); } const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text)); const sections: ExtractedSection[] = []; let empty = 0; let chars = 0; for (let i = 0; i < total; i++) { const text = cleaned[i]; if (!text) { empty++; continue; } chars += text.length; sections.push({ text, page: i + 1 }); } const warnings: string[] = []; if (empty > 0) { warnings.push( empty === total ? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.` : `${empty} of ${total} pages had no extractable text.`, ); } if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`); const result: ExtractResult = { kind: 'pdf', sections, pageCount: total, warnings, needsOcr: chars / total < OCR_CHARS_PER_PAGE, }; if (title) result.title = title; return result; } catch (error) { throw toExtractionError(error); } finally { try { await pdf.loadingTask.destroy(); } catch { // Nothing useful to do. } } }