V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
249 lines
8.5 KiB
TypeScript
249 lines
8.5 KiB
TypeScript
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
|
|
import { getDocumentProxy } from 'unpdf';
|
|
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
|
|
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
|
|
export interface PdfContext {
|
|
onProgress?: (done: number, total: number) => void;
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
/** Below this average of characters per page the PDF is treated as a scan. */
|
|
const OCR_CHARS_PER_PAGE = 25;
|
|
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
|
|
const CLEANUP_EVERY = 40;
|
|
|
|
interface TextItemLike {
|
|
str?: unknown;
|
|
hasEOL?: unknown;
|
|
transform?: unknown;
|
|
height?: unknown;
|
|
}
|
|
|
|
interface Line {
|
|
text: string;
|
|
y: number | null;
|
|
height: number;
|
|
}
|
|
|
|
/**
|
|
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
|
|
* order (which keeps multi-column pages readable) with `hasEOL` marking line
|
|
* ends; the vertical gaps between lines are then used to restore paragraph
|
|
* breaks, which pdf.js does not report.
|
|
*/
|
|
export function textItemsToText(items: readonly TextItemLike[]): string {
|
|
const lines: Line[] = [];
|
|
let text = '';
|
|
let y: number | null = null;
|
|
let height = 0;
|
|
const endLine = () => {
|
|
lines.push({ text, y, height });
|
|
text = '';
|
|
y = null;
|
|
height = 0;
|
|
};
|
|
for (const item of items) {
|
|
if (!item || typeof item.str !== 'string') continue;
|
|
const str = item.str;
|
|
if (str.length > 0) {
|
|
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
|
|
const t = item.transform as number[];
|
|
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
|
|
if (upright && Number.isFinite(t[5])) {
|
|
y = t[5];
|
|
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
|
|
height = Number.isFinite(h) ? h : 0;
|
|
}
|
|
}
|
|
text += str;
|
|
}
|
|
if (item.hasEOL === true) endLine();
|
|
}
|
|
if (text.length > 0) endLine();
|
|
|
|
// Typical line advance on this page (median of the downward steps).
|
|
const advances: number[] = [];
|
|
for (let i = 1; i < lines.length; i++) {
|
|
const a = lines[i - 1];
|
|
const b = lines[i];
|
|
if (a.y === null || b.y === null) continue;
|
|
const dy = a.y - b.y;
|
|
const ref = Math.max(a.height, b.height, 1);
|
|
if (dy > 0 && dy < ref * 4) advances.push(dy);
|
|
}
|
|
advances.sort((p, q) => p - q);
|
|
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
|
|
|
|
let out = '';
|
|
let prev: Line | null = null;
|
|
for (const line of lines) {
|
|
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
|
|
if (!content) {
|
|
// An empty line is an explicit gap.
|
|
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
|
|
continue;
|
|
}
|
|
if (out && !out.endsWith('\n')) {
|
|
let gap = false;
|
|
if (prev && prev.y !== null && line.y !== null) {
|
|
const dy = prev.y - line.y;
|
|
const ref = Math.max(prev.height, line.height, 1);
|
|
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
|
|
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
|
|
else gap = dy > ref * 1.9;
|
|
}
|
|
out += gap ? '\n\n' : '\n';
|
|
}
|
|
out += content;
|
|
prev = line;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
|
|
|
|
export function tidyTitle(raw: unknown): string | undefined {
|
|
if (typeof raw !== 'string') return undefined;
|
|
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
|
|
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
|
|
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
|
|
return title;
|
|
}
|
|
|
|
function errorName(error: unknown): string {
|
|
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
|
|
}
|
|
|
|
function toExtractionError(error: unknown): ExtractionError {
|
|
if (error instanceof ExtractionError) return error;
|
|
const name = errorName(error);
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (name === 'PasswordException' || /password/i.test(message)) {
|
|
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
|
|
}
|
|
if (
|
|
name === 'InvalidPDFException' ||
|
|
name === 'FormatError' ||
|
|
name === 'XRefParseException' ||
|
|
name === 'XRefEntryException' ||
|
|
name === 'MissingDataException' ||
|
|
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
|
|
) {
|
|
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
|
|
}
|
|
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
|
|
}
|
|
|
|
function cancelled(signal: AbortSignal): ExtractionError {
|
|
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
|
|
}
|
|
|
|
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
|
|
const { onProgress, signal } = ctx;
|
|
if (signal?.aborted) throw cancelled(signal);
|
|
|
|
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
|
|
try {
|
|
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
|
|
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
|
|
pdf = await getDocumentProxy(bytes.slice(), options);
|
|
} catch (error) {
|
|
throw toExtractionError(error);
|
|
}
|
|
|
|
try {
|
|
const total = pdf.numPages;
|
|
if (!Number.isFinite(total) || total < 1) {
|
|
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
|
|
}
|
|
|
|
let title: string | undefined;
|
|
try {
|
|
const meta = await pdf.getMetadata();
|
|
const info = (meta?.info ?? {}) as Record<string, unknown>;
|
|
title = tidyTitle(info.Title);
|
|
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
|
|
} catch {
|
|
// Metadata is optional.
|
|
}
|
|
|
|
const pageTexts: string[] = new Array(total).fill('');
|
|
let failedPages = 0;
|
|
let lastYield = Date.now();
|
|
for (let n = 1; n <= total; n++) {
|
|
if (signal?.aborted) throw cancelled(signal);
|
|
try {
|
|
const page = await pdf.getPage(n);
|
|
try {
|
|
const content = await page.getTextContent();
|
|
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
|
|
} finally {
|
|
page.cleanup();
|
|
}
|
|
} catch (error) {
|
|
if (errorName(error) === 'PasswordException') throw error;
|
|
failedPages++;
|
|
}
|
|
if (n % CLEANUP_EVERY === 0) {
|
|
try {
|
|
await pdf.cleanup();
|
|
} catch {
|
|
// Best effort.
|
|
}
|
|
}
|
|
onProgress?.(n, total);
|
|
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
|
|
if (Date.now() - lastYield > 20) {
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
lastYield = Date.now();
|
|
}
|
|
}
|
|
|
|
if (failedPages === total) {
|
|
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
|
|
}
|
|
|
|
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
|
|
const sections: ExtractedSection[] = [];
|
|
let empty = 0;
|
|
let chars = 0;
|
|
for (let i = 0; i < total; i++) {
|
|
const text = cleaned[i];
|
|
if (!text) {
|
|
empty++;
|
|
continue;
|
|
}
|
|
chars += text.length;
|
|
sections.push({ text, page: i + 1 });
|
|
}
|
|
|
|
const warnings: string[] = [];
|
|
if (empty > 0) {
|
|
warnings.push(
|
|
empty === total
|
|
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
|
|
: `${empty} of ${total} pages had no extractable text.`,
|
|
);
|
|
}
|
|
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
|
|
|
|
const result: ExtractResult = {
|
|
kind: 'pdf',
|
|
sections,
|
|
pageCount: total,
|
|
warnings,
|
|
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
|
|
};
|
|
if (title) result.title = title;
|
|
return result;
|
|
} catch (error) {
|
|
throw toExtractionError(error);
|
|
} finally {
|
|
try {
|
|
await pdf.loadingTask.destroy();
|
|
} catch {
|
|
// Nothing useful to do.
|
|
}
|
|
}
|
|
}
|