Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
249
server/documents/pdf.ts
Normal file
249
server/documents/pdf.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
|
||||
import { getDocumentProxy } from 'unpdf';
|
||||
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
|
||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
||||
|
||||
export interface PdfContext {
|
||||
onProgress?: (done: number, total: number) => void;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** Below this average of characters per page the PDF is treated as a scan. */
|
||||
const OCR_CHARS_PER_PAGE = 25;
|
||||
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
|
||||
const CLEANUP_EVERY = 40;
|
||||
|
||||
interface TextItemLike {
|
||||
str?: unknown;
|
||||
hasEOL?: unknown;
|
||||
transform?: unknown;
|
||||
height?: unknown;
|
||||
}
|
||||
|
||||
interface Line {
|
||||
text: string;
|
||||
y: number | null;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
|
||||
* order (which keeps multi-column pages readable) with `hasEOL` marking line
|
||||
* ends; the vertical gaps between lines are then used to restore paragraph
|
||||
* breaks, which pdf.js does not report.
|
||||
*/
|
||||
export function textItemsToText(items: readonly TextItemLike[]): string {
|
||||
const lines: Line[] = [];
|
||||
let text = '';
|
||||
let y: number | null = null;
|
||||
let height = 0;
|
||||
const endLine = () => {
|
||||
lines.push({ text, y, height });
|
||||
text = '';
|
||||
y = null;
|
||||
height = 0;
|
||||
};
|
||||
for (const item of items) {
|
||||
if (!item || typeof item.str !== 'string') continue;
|
||||
const str = item.str;
|
||||
if (str.length > 0) {
|
||||
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
|
||||
const t = item.transform as number[];
|
||||
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
|
||||
if (upright && Number.isFinite(t[5])) {
|
||||
y = t[5];
|
||||
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
|
||||
height = Number.isFinite(h) ? h : 0;
|
||||
}
|
||||
}
|
||||
text += str;
|
||||
}
|
||||
if (item.hasEOL === true) endLine();
|
||||
}
|
||||
if (text.length > 0) endLine();
|
||||
|
||||
// Typical line advance on this page (median of the downward steps).
|
||||
const advances: number[] = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const a = lines[i - 1];
|
||||
const b = lines[i];
|
||||
if (a.y === null || b.y === null) continue;
|
||||
const dy = a.y - b.y;
|
||||
const ref = Math.max(a.height, b.height, 1);
|
||||
if (dy > 0 && dy < ref * 4) advances.push(dy);
|
||||
}
|
||||
advances.sort((p, q) => p - q);
|
||||
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
|
||||
|
||||
let out = '';
|
||||
let prev: Line | null = null;
|
||||
for (const line of lines) {
|
||||
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
|
||||
if (!content) {
|
||||
// An empty line is an explicit gap.
|
||||
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
|
||||
continue;
|
||||
}
|
||||
if (out && !out.endsWith('\n')) {
|
||||
let gap = false;
|
||||
if (prev && prev.y !== null && line.y !== null) {
|
||||
const dy = prev.y - line.y;
|
||||
const ref = Math.max(prev.height, line.height, 1);
|
||||
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
|
||||
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
|
||||
else gap = dy > ref * 1.9;
|
||||
}
|
||||
out += gap ? '\n\n' : '\n';
|
||||
}
|
||||
out += content;
|
||||
prev = line;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
|
||||
|
||||
export function tidyTitle(raw: unknown): string | undefined {
|
||||
if (typeof raw !== 'string') return undefined;
|
||||
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
|
||||
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
|
||||
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
|
||||
return title;
|
||||
}
|
||||
|
||||
function errorName(error: unknown): string {
|
||||
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
|
||||
}
|
||||
|
||||
function toExtractionError(error: unknown): ExtractionError {
|
||||
if (error instanceof ExtractionError) return error;
|
||||
const name = errorName(error);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (name === 'PasswordException' || /password/i.test(message)) {
|
||||
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
|
||||
}
|
||||
if (
|
||||
name === 'InvalidPDFException' ||
|
||||
name === 'FormatError' ||
|
||||
name === 'XRefParseException' ||
|
||||
name === 'XRefEntryException' ||
|
||||
name === 'MissingDataException' ||
|
||||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
|
||||
) {
|
||||
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
|
||||
}
|
||||
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
|
||||
}
|
||||
|
||||
function cancelled(signal: AbortSignal): ExtractionError {
|
||||
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
|
||||
}
|
||||
|
||||
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
|
||||
const { onProgress, signal } = ctx;
|
||||
if (signal?.aborted) throw cancelled(signal);
|
||||
|
||||
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
|
||||
try {
|
||||
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
|
||||
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
|
||||
pdf = await getDocumentProxy(bytes.slice(), options);
|
||||
} catch (error) {
|
||||
throw toExtractionError(error);
|
||||
}
|
||||
|
||||
try {
|
||||
const total = pdf.numPages;
|
||||
if (!Number.isFinite(total) || total < 1) {
|
||||
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
|
||||
}
|
||||
|
||||
let title: string | undefined;
|
||||
try {
|
||||
const meta = await pdf.getMetadata();
|
||||
const info = (meta?.info ?? {}) as Record<string, unknown>;
|
||||
title = tidyTitle(info.Title);
|
||||
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
|
||||
} catch {
|
||||
// Metadata is optional.
|
||||
}
|
||||
|
||||
const pageTexts: string[] = new Array(total).fill('');
|
||||
let failedPages = 0;
|
||||
let lastYield = Date.now();
|
||||
for (let n = 1; n <= total; n++) {
|
||||
if (signal?.aborted) throw cancelled(signal);
|
||||
try {
|
||||
const page = await pdf.getPage(n);
|
||||
try {
|
||||
const content = await page.getTextContent();
|
||||
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
|
||||
} finally {
|
||||
page.cleanup();
|
||||
}
|
||||
} catch (error) {
|
||||
if (errorName(error) === 'PasswordException') throw error;
|
||||
failedPages++;
|
||||
}
|
||||
if (n % CLEANUP_EVERY === 0) {
|
||||
try {
|
||||
await pdf.cleanup();
|
||||
} catch {
|
||||
// Best effort.
|
||||
}
|
||||
}
|
||||
onProgress?.(n, total);
|
||||
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
|
||||
if (Date.now() - lastYield > 20) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
lastYield = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
if (failedPages === total) {
|
||||
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
|
||||
}
|
||||
|
||||
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
|
||||
const sections: ExtractedSection[] = [];
|
||||
let empty = 0;
|
||||
let chars = 0;
|
||||
for (let i = 0; i < total; i++) {
|
||||
const text = cleaned[i];
|
||||
if (!text) {
|
||||
empty++;
|
||||
continue;
|
||||
}
|
||||
chars += text.length;
|
||||
sections.push({ text, page: i + 1 });
|
||||
}
|
||||
|
||||
const warnings: string[] = [];
|
||||
if (empty > 0) {
|
||||
warnings.push(
|
||||
empty === total
|
||||
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
|
||||
: `${empty} of ${total} pages had no extractable text.`,
|
||||
);
|
||||
}
|
||||
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
|
||||
|
||||
const result: ExtractResult = {
|
||||
kind: 'pdf',
|
||||
sections,
|
||||
pageCount: total,
|
||||
warnings,
|
||||
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
|
||||
};
|
||||
if (title) result.title = title;
|
||||
return result;
|
||||
} catch (error) {
|
||||
throw toExtractionError(error);
|
||||
} finally {
|
||||
try {
|
||||
await pdf.loadingTask.destroy();
|
||||
} catch {
|
||||
// Nothing useful to do.
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue