Study/server/documents/pdf.ts
Elijah 3e74ecd843 Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 08:16:41 -07:00

249 lines
8.5 KiB
TypeScript

/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
import { getDocumentProxy } from 'unpdf';
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
export interface PdfContext {
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
/** Below this average of characters per page the PDF is treated as a scan. */
const OCR_CHARS_PER_PAGE = 25;
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
const CLEANUP_EVERY = 40;
interface TextItemLike {
str?: unknown;
hasEOL?: unknown;
transform?: unknown;
height?: unknown;
}
interface Line {
text: string;
y: number | null;
height: number;
}
/**
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
* order (which keeps multi-column pages readable) with `hasEOL` marking line
* ends; the vertical gaps between lines are then used to restore paragraph
* breaks, which pdf.js does not report.
*/
export function textItemsToText(items: readonly TextItemLike[]): string {
const lines: Line[] = [];
let text = '';
let y: number | null = null;
let height = 0;
const endLine = () => {
lines.push({ text, y, height });
text = '';
y = null;
height = 0;
};
for (const item of items) {
if (!item || typeof item.str !== 'string') continue;
const str = item.str;
if (str.length > 0) {
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
const t = item.transform as number[];
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
if (upright && Number.isFinite(t[5])) {
y = t[5];
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
height = Number.isFinite(h) ? h : 0;
}
}
text += str;
}
if (item.hasEOL === true) endLine();
}
if (text.length > 0) endLine();
// Typical line advance on this page (median of the downward steps).
const advances: number[] = [];
for (let i = 1; i < lines.length; i++) {
const a = lines[i - 1];
const b = lines[i];
if (a.y === null || b.y === null) continue;
const dy = a.y - b.y;
const ref = Math.max(a.height, b.height, 1);
if (dy > 0 && dy < ref * 4) advances.push(dy);
}
advances.sort((p, q) => p - q);
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
let out = '';
let prev: Line | null = null;
for (const line of lines) {
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
if (!content) {
// An empty line is an explicit gap.
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
continue;
}
if (out && !out.endsWith('\n')) {
let gap = false;
if (prev && prev.y !== null && line.y !== null) {
const dy = prev.y - line.y;
const ref = Math.max(prev.height, line.height, 1);
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
else gap = dy > ref * 1.9;
}
out += gap ? '\n\n' : '\n';
}
out += content;
prev = line;
}
return out;
}
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
export function tidyTitle(raw: unknown): string | undefined {
if (typeof raw !== 'string') return undefined;
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
return title;
}
function errorName(error: unknown): string {
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
}
function toExtractionError(error: unknown): ExtractionError {
if (error instanceof ExtractionError) return error;
const name = errorName(error);
const message = error instanceof Error ? error.message : String(error);
if (name === 'PasswordException' || /password/i.test(message)) {
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
}
if (
name === 'InvalidPDFException' ||
name === 'FormatError' ||
name === 'XRefParseException' ||
name === 'XRefEntryException' ||
name === 'MissingDataException' ||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
) {
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
}
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
}
function cancelled(signal: AbortSignal): ExtractionError {
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
}
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
const { onProgress, signal } = ctx;
if (signal?.aborted) throw cancelled(signal);
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
try {
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
pdf = await getDocumentProxy(bytes.slice(), options);
} catch (error) {
throw toExtractionError(error);
}
try {
const total = pdf.numPages;
if (!Number.isFinite(total) || total < 1) {
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
}
let title: string | undefined;
try {
const meta = await pdf.getMetadata();
const info = (meta?.info ?? {}) as Record<string, unknown>;
title = tidyTitle(info.Title);
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
} catch {
// Metadata is optional.
}
const pageTexts: string[] = new Array(total).fill('');
let failedPages = 0;
let lastYield = Date.now();
for (let n = 1; n <= total; n++) {
if (signal?.aborted) throw cancelled(signal);
try {
const page = await pdf.getPage(n);
try {
const content = await page.getTextContent();
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
} finally {
page.cleanup();
}
} catch (error) {
if (errorName(error) === 'PasswordException') throw error;
failedPages++;
}
if (n % CLEANUP_EVERY === 0) {
try {
await pdf.cleanup();
} catch {
// Best effort.
}
}
onProgress?.(n, total);
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
if (failedPages === total) {
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
}
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
const sections: ExtractedSection[] = [];
let empty = 0;
let chars = 0;
for (let i = 0; i < total; i++) {
const text = cleaned[i];
if (!text) {
empty++;
continue;
}
chars += text.length;
sections.push({ text, page: i + 1 });
}
const warnings: string[] = [];
if (empty > 0) {
warnings.push(
empty === total
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
: `${empty} of ${total} pages had no extractable text.`,
);
}
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
const result: ExtractResult = {
kind: 'pdf',
sections,
pageCount: total,
warnings,
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
};
if (title) result.title = title;
return result;
} catch (error) {
throw toExtractionError(error);
} finally {
try {
await pdf.loadingTask.destroy();
} catch {
// Nothing useful to do.
}
}
}