Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

514
server/documents/chunk.ts Normal file
View file

@ -0,0 +1,514 @@
/**
* Text normalisation, token estimation and retrieval chunking.
* Pure functions, no dependencies. Everything here is linear-time on its input.
*/
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
// ---------------------------------------------------------------------------
// Token estimation
// ---------------------------------------------------------------------------
/**
* Un-rounded token estimate. Sub-additive under concatenation
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
*/
function rawTokens(text: string): number {
let total = 0;
let prevWhitespace = false;
for (let i = 0; i < text.length; i++) {
const c = text.charCodeAt(i);
if (c === 32 || (c >= 9 && c <= 13)) {
// A run of whitespace costs the same as one character of prose.
if (!prevWhitespace) total += 0.25;
prevWhitespace = true;
continue;
}
prevWhitespace = false;
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
else if (c >= 48 && c <= 57) total += 0.4;
else if (c < 128) total += 0.5;
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
else total += 0.5; // accented Latin, Greek, symbols
}
return total;
}
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
export function estimateTokens(text: string): number {
if (!text) return 0;
return Math.ceil(rawTokens(text));
}
// ---------------------------------------------------------------------------
// cleanText
// ---------------------------------------------------------------------------
const LIGATURES: Record<string, string> = {
'ff': 'ff',
'fi': 'fi',
'fl': 'fl',
'ffi': 'ffi',
'ffl': 'ffl',
'ſt': 'st',
'st': 'st',
};
/** First halves that normally form a real hyphenated compound ("well-known"). */
const HYPHEN_KEEP_LEFT = new Set([
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
]);
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
const HYPHEN_KEEP_RIGHT = new Set([
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
]);
function isLower(code: number): boolean {
return code >= 97 && code <= 122;
}
function isLetter(code: number): boolean {
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
}
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
function joinHyphenatedLineBreaks(text: string): string {
if (!text.includes('-\n')) return text;
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
let start = offset;
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
const left = whole.slice(start, offset);
if (left.length < 2) return match;
if (start > 0) {
const before = whole.charCodeAt(start - 1);
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
}
const rightStart = offset + match.length;
let rightEnd = rightStart;
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
const right = whole.slice(rightStart, rightEnd);
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
return '';
});
}
/**
* Normalises extracted text: newlines, unicode spaces, control characters,
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
* a line break. Idempotent.
*/
export function cleanText(text: string): string {
if (!text) return '';
let t = text;
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
t = t
.split('\n')
.map((line) => line.trimEnd())
.join('\n');
t = joinHyphenatedLineBreaks(t);
t = t.replace(/\n{3,}/g, '\n\n');
return t.trim();
}
// ---------------------------------------------------------------------------
// Repeated page headers / footers
// ---------------------------------------------------------------------------
function edgeLineKey(line: string): string | null {
const trimmed = line.trim();
if (!trimmed || trimmed.length > 100) return null;
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
}
/**
* Removes running headers/footers and page numbers from per-page text, but only
* when the same first or last line (digits ignored) shows up on at least 60% of
* the pages that have text, and there are at least four such pages. Chapter
* headings that change through a book are therefore left alone.
*/
export function stripRepeatedPageLines(pages: string[]): string[] {
let current = pages.map((page) => page.split('\n'));
for (let pass = 0; pass < 2; pass++) {
const firstIdx: number[] = [];
const lastIdx: number[] = [];
const topCounts = new Map<string, number>();
const bottomCounts = new Map<string, number>();
let textPages = 0;
for (let p = 0; p < current.length; p++) {
const lines = current[p];
let first = 0;
while (first < lines.length && !lines[first].trim()) first++;
let last = lines.length - 1;
while (last >= 0 && !lines[last].trim()) last--;
firstIdx.push(first < lines.length ? first : -1);
lastIdx.push(last);
if (last < 0) continue;
textPages++;
const top = edgeLineKey(lines[first]);
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
}
}
if (textPages < 4) break;
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
let removed = false;
current = current.map((lines, p) => {
const first = firstIdx[p];
const last = lastIdx[p];
if (last < 0) return lines;
let from = 0;
let to = lines.length;
const top = edgeLineKey(lines[first]);
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
}
if (from === 0 && to === lines.length) return lines;
removed = true;
return lines.slice(from, to);
});
if (!removed) break;
}
return current.map((lines) => lines.join('\n'));
}
// ---------------------------------------------------------------------------
// Chunking
// ---------------------------------------------------------------------------
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
interface Atom {
text: string;
raw: number;
sep: string;
sepRaw: number;
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
strength: number;
}
const WHITESPACE_RAW = 0.25;
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
function sepRawOf(sep: string): number {
return sep ? WHITESPACE_RAW : 0;
}
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
let start = 0;
let acc = 0;
let first = true;
const push = (end: number) => {
const text = word.slice(start, end);
out.push({
text,
raw: rawTokens(text),
sep: first ? sep : '',
sepRaw: first ? sepRawOf(sep) : 0,
strength: first ? strength : -1,
});
first = false;
start = end;
acc = 0;
};
for (let i = 0; i < word.length; i++) {
const c = word.charCodeAt(i);
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
acc += weight;
}
if (start < word.length) push(word.length);
}
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
const raw = rawTokens(sentence);
if (raw <= atomMax) {
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
return;
}
// Oversized sentence (or table row, or blob): fall back to groups of words.
const groupMax = Math.max(1, Math.min(atomMax, 24));
const words = sentence.split(/[ \t]+/);
// Keep leading indentation attached to the first word.
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
let group = '';
let groupRaw = 0;
let first = true;
const flush = () => {
if (!group) return;
out.push({
text: group,
raw: rawTokens(group),
sep: first ? sep : ' ',
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
strength: first ? strength : 0,
});
first = false;
group = '';
groupRaw = 0;
};
let pendingIndent = indent;
for (const word of words) {
if (!word) continue;
const piece = pendingIndent + word;
pendingIndent = '';
const wordRaw = rawTokens(piece);
if (wordRaw > atomMax) {
flush();
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
first = false;
continue;
}
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
group = group ? `${group} ${piece}` : piece;
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
}
flush();
}
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
function atomize(text: string, atomMax: number): Atom[] {
const atoms: Atom[] = [];
let prevLine = '';
for (const para of text.split(/\n{2,}/)) {
let firstInPara = true;
for (const rawLine of para.split('\n')) {
const line = rawLine.trimEnd();
if (!line.trim()) continue;
let sep = '';
let strength = 0;
if (atoms.length > 0) {
const afterHeading = HEADING_LINE.test(prevLine);
const isHeading = HEADING_LINE.test(line);
if (firstInPara) {
sep = '\n\n';
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
} else {
sep = '\n';
if (isHeading) strength = 4;
else if (afterHeading) strength = -0.5;
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
else strength = 0.5;
}
}
firstInPara = false;
prevLine = line;
if (rawTokens(line) <= Math.min(atomMax, 12)) {
// Short line: not worth looking for sentences.
pushSentence(line, atomMax, atoms, sep, strength);
continue;
}
let firstSentence = true;
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
if (!sentence) continue;
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
firstSentence = false;
}
}
}
return atoms;
}
function joinAtoms(atoms: Atom[], from: number, to: number): string {
let text = '';
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
return text;
}
/** Trailing whole words of `text` worth at most `budget` tokens. */
function tailWords(text: string, budget: number): string {
let end = text.length;
let cut = text.length;
let acc = 0;
while (end > 0) {
let wordEnd = end;
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
if (wordEnd === 0) break;
const space = text.lastIndexOf(' ', wordEnd - 1);
const word = text.slice(space + 1, wordEnd);
const cost = rawTokens(word) + WHITESPACE_RAW;
if (acc + cost > budget) break;
acc += cost;
cut = space + 1;
end = space < 0 ? 0 : space;
}
return text.slice(cut).trim();
}
/**
* Splits one oversized section into balanced pieces of at most `bodyBudget`
* tokens, cutting at the strongest boundary near the balanced size, and
* prefixing each later piece with up to `overlap` tokens of trailing context.
*/
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
const atomMax = Math.max(1, bodyBudget / 4);
const atoms = atomize(text, atomMax);
const n = atoms.length;
const pieces: string[] = [];
if (n === 0) return pieces;
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
const suffix = new Float64Array(n + 1);
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
let start = 0;
let prevStart = -1;
while (start < n) {
const remaining = suffix[start] - atoms[start].sepRaw;
let end: number;
if (remaining <= bodyBudget) {
end = n;
} else {
const goal = remaining / Math.ceil(remaining / bodyBudget);
const minFill = goal * 0.6;
let acc = 0;
let best = -1;
let bestStrength = -Infinity;
let bestDistance = Infinity;
let i = start;
for (; i < n; i++) {
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
if (i > start && acc + add > bodyBudget) break;
acc += add;
if (i + 1 < n && acc >= minFill) {
const strength = atoms[i + 1].strength;
const distance = Math.abs(acc - goal);
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
best = i;
bestStrength = strength;
bestDistance = distance;
}
}
}
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
}
let prefix = '';
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
let from = start;
let acc = 0;
while (from - 1 > prevStart) {
const candidate = atoms[from - 1];
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
if (acc + cost > overlap) break;
acc += cost;
from--;
if (candidate.strength >= 4) break; // do not reach back past a heading
}
if (from < start) prefix = joinAtoms(atoms, from, start);
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
}
const body = joinAtoms(atoms, start, end);
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
prevStart = start;
start = end;
}
return pieces;
}
function positiveInt(value: number | undefined, fallback: number, min: number): number {
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
return Math.max(min, Math.floor(value));
}
interface Pending {
text: string;
heading: string;
page: number | null;
}
/**
* Builds retrieval chunks from the sections of ONE document.
*
* - Consecutive small sections (short slides, short pages) are packed together
* up to `targetTokens`; the chunk keeps the first section's heading and the
* full page range.
* - A section that fits comfortably under the ceiling stays whole so chunks
* line up with pages/slides.
* - Larger sections are split into balanced pieces at paragraph, then
* sentence, then word boundaries, with `overlapTokens` of repeated context
* between consecutive pieces of the same section only.
*/
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
const target = positiveInt(opts.targetTokens, 450, 8);
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
/** Sections up to this size are kept in one piece. */
const wholeLimit = Math.floor(target + (max - target) / 2);
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
const chunks: DocChunk[] = [];
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
const trimmed = text.trim();
if (!trimmed) return;
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
};
let pending: Pending[] = [];
let pendingRaw = 0;
const flush = () => {
if (pending.length === 0) return;
let pageStart: number | null = null;
let pageEnd: number | null = null;
for (const item of pending) {
if (item.page === null) continue;
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
}
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
pending = [];
pendingRaw = 0;
};
let lastHeading = '';
for (const section of sections ?? []) {
if (!section || typeof section.text !== 'string') continue;
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
if (ownHeading) lastHeading = ownHeading;
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
if (!text) continue;
const heading = lastHeading;
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
const raw = rawTokens(text);
if (raw > wholeLimit) {
flush();
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
continue;
}
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
pending.push({ text, heading, page });
}
flush();
return chunks;
}

View file

@ -0,0 +1,420 @@
import { strToU8, zipSync } from 'fflate';
import { describe, expect, it } from 'vitest';
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
import { detectKind, extractDocument } from './extract.ts';
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
import { ExtractionError, type ExtractedSection } from './types.ts';
const enc = (s: string) => new TextEncoder().encode(s);
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
async function codeOf(promise: Promise<unknown>): Promise<string> {
try {
await promise;
} catch (e) {
expect(e).toBeInstanceOf(ExtractionError);
return (e as ExtractionError).code;
}
throw new Error('expected rejection');
}
describe('detectKind', () => {
const pdf = buildPdf([{ lines: ['hello'] }]);
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
const pptx = buildPptx([{ file: 1, title: 'T' }]);
it('detects by magic bytes', () => {
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
});
it('trusts magic bytes over a wrong extension', () => {
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
});
it('uses the extension for text and markdown', () => {
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
});
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
});
});
describe('extractDocument: PDF', () => {
it('extracts multi-page text with page numbers, count and title', async () => {
const bytes = buildPdf(
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
{ title: 'Cardiology Notes' },
);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.kind).toBe('pdf');
expect(r.pageCount).toBe(2);
expect(r.title).toBe('Cardiology Notes');
expect(r.needsOcr).toBe(false);
expect(r.warnings).toEqual([]);
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
expect(r.sections[0].text).toContain('Heart failure overview');
expect(r.sections[1].text).toContain('Digoxin');
});
it('warns about a page without text but keeps page numbers', async () => {
const long = 'Readable text on this page, long enough to avoid OCR. ';
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.pageCount).toBe(3);
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
});
it('flags an all-empty scanned PDF as needing OCR', async () => {
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
expect(r.needsOcr).toBe(true);
expect(r.sections).toEqual([]);
expect(r.pageCount).toBe(2);
expect(r.warnings.join(' ')).toMatch(/scanned/);
});
it('reports progress per page', async () => {
const seen: Array<[number, number]> = [];
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
expect(seen).toEqual([
[1, 2],
[2, 2],
]);
});
});
describe('extractDocument: DOCX', () => {
it('turns headings into sections and keeps lists and tables readable', async () => {
const bytes = buildDocx(
[
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
{ type: 'p', text: 'Intro paragraph about drugs.' },
{ type: 'bullet', text: 'Beta blockers' },
{ type: 'bullet', text: 'ACE inhibitors' },
{ type: 'heading', level: 2, text: 'Dosing' },
{ type: 'number', text: 'Check heart rate' },
{ type: 'number', text: 'Give the dose' },
{
type: 'table',
rows: [
['Drug', 'Dose'],
['Digoxin', '0.25 mg'],
],
},
],
{ title: 'Drug Guide' },
);
const r = await extractDocument({ bytes, filename: 'a.docx' });
expect(r.kind).toBe('docx');
expect(r.title).toBe('Drug Guide');
expect(r.needsOcr).toBe(false);
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
const first = r.sections[0].text;
expect(first).toContain('Intro paragraph');
expect(first).toContain('Beta blockers');
expect(first).toContain('ACE inhibitors');
const second = r.sections[1].text;
expect(second).toContain('Check heart rate');
expect(second).toContain('Digoxin');
expect(second).toContain('0.25 mg');
});
it('rejects a docx with no text as empty', async () => {
const bytes = buildDocx([{ type: 'p', text: '' }]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
});
});
describe('extractDocument: PPTX', () => {
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
const bytes = buildPptx(
[
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
{ file: 2 },
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
],
{ title: 'Deck Title' },
);
const r = await extractDocument({ bytes, filename: 'a.pptx' });
expect(r.kind).toBe('pptx');
expect(r.title).toBe('Deck Title');
expect(r.pageCount).toBe(4);
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
expect(r.sections[0].text).toContain('Welcome');
expect(r.sections[0].text).toContain('Sub point');
expect(r.sections[0].text).toContain('Say hello slowly');
expect(r.sections[1].text).toContain('Free text box content here');
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
});
it('reads tables', async () => {
const table = [
['A', 'B'],
['x', 'y'],
];
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
expect(allText(r)).toContain('x');
expect(allText(r)).toContain('B');
});
it('warns when most slides have no text', async () => {
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
});
});
describe('extractDocument: text, markdown, image', () => {
it('splits markdown by heading and ignores headings inside code fences', async () => {
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
expect(r.kind).toBe('markdown');
expect(r.title).toBe('Title');
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
expect(r.sections[1].text).toContain('# not a heading');
expect(r.needsOcr).toBe(false);
});
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
expect(r.kind).toBe('text');
expect(allText(r)).toContain('line two');
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
expect(allText(r2)).toBe('café');
expect(r2.warnings.length).toBe(1);
});
it('groups long plain text into several bounded sections', async () => {
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
expect(r.sections.length).toBeGreaterThan(1);
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
});
it('returns images with needsOcr and no sections', async () => {
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
});
});
describe('extractDocument: errors', () => {
it('rejects an empty file', async () => {
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
});
it('rejects whitespace-only text as empty', async () => {
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
});
it('rejects unsupported types and legacy .doc', async () => {
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
});
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
const marker = Buffer.from('EncryptedPackage', 'utf16le');
const bytes = Uint8Array.from([...OLE, ...marker]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
});
it('rejects corrupt PDFs', async () => {
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
});
it('rejects an encrypted PDF', async () => {
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
});
it('rejects a corrupt zip posing as docx/pptx', async () => {
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
const broken = good.slice(0, Math.floor(good.length / 2));
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a pptx zip that has no readable slides', async () => {
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a zip bomb whose headers lie about entry size', async () => {
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
expect(['too_large', 'corrupt']).toContain(code);
}, 30000);
it('honours an already-aborted signal', async () => {
const ac = new AbortController();
ac.abort();
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
});
});
describe('chunkSections', () => {
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
page: i + 1,
}));
const chunks = chunkSections(sections, opts);
expect(chunks.length).toBeGreaterThan(12);
chunks.forEach((c, i) => {
expect(c.seq).toBe(i);
expect(c.tokens).toBe(estimateTokens(c.text));
expect(c.tokens).toBeLessThanOrEqual(150);
});
});
it('packs small sections together with correct page range and first heading', () => {
const sections: ExtractedSection[] = [
{ text: 'Alpha text.', page: 1, heading: 'First' },
{ text: 'Beta text.', page: 2 },
{ text: 'Gamma text.', page: 3, heading: 'Third' },
];
const chunks = chunkSections(sections, opts);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
expect(chunks[0].text).toContain('Gamma');
});
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(c.pageStart).not.toBeNull();
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
}
expect(chunks[0].pageStart).toBe(1);
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
});
it('uses null pages and empty heading when sections have none', () => {
const [c] = chunkSections([{ text: 'No pages here.' }]);
expect(c.pageStart).toBeNull();
expect(c.pageEnd).toBeNull();
expect(c.heading).toBe('');
});
it('splits oversized sections without cutting words', () => {
const text = words(800, 'term');
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
expect(chunks.length).toBeGreaterThan(3);
const vocabulary = new Set(text.split(' '));
for (const c of chunks) {
expect(c.pageStart).toBe(5);
expect(c.pageEnd).toBe(5);
expect(c.heading).toBe('Big');
expect(c.tokens).toBeLessThanOrEqual(150);
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
}
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
expect(seen.size).toBe(800);
});
it('survives a giant unbroken string within the limit', () => {
const blob = 'x'.repeat(200_000);
const t0 = Date.now();
const chunks = chunkSections([{ text: blob }], opts);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
});
it('does not split surrogate pairs in unbroken emoji runs', () => {
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(wellFormed(c.text)).toBe(true);
expect(c.tokens).toBeLessThanOrEqual(150);
}
});
it('is deterministic', () => {
const sections = [
{ text: words(900), page: 1, heading: 'H' },
{ text: 'Short one.', page: 2 },
];
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
});
it('drops empty and whitespace-only sections', () => {
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
const chunks = chunkSections([
{ text: ' ', page: 1 },
{ text: 'Real content.', page: 2 },
{ text: '\n', page: 3 },
]);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
});
it('chunks about 2 MB of text quickly', () => {
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
const t0 = Date.now();
const chunks = chunkSections([{ text }]);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(100);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
});
});
describe('cleanText', () => {
it('joins words hyphenated across a line break', () => {
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
});
it('keeps real compounds across a line break', () => {
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
});
it('collapses 3+ newlines to a paragraph break', () => {
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
expect(cleanText('a\n\nb')).toBe('a\n\nb');
});
it('strips control characters but keeps tabs and newlines', () => {
expect(cleanText('he\u0000l\u0007lo​ ­world\ttab\nnext')).toBe('hello world\ttab\nnext');
});
it('normalises CRLF and trims', () => {
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
expect(cleanText('')).toBe('');
});
it('is idempotent', () => {
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny  z');
expect(cleanText(once)).toBe(once);
});
});

105
server/documents/docx.ts Normal file
View file

@ -0,0 +1,105 @@
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
import { zipSync } from 'fflate';
import mammoth from 'mammoth';
import { cleanText } from './chunk.ts';
import { htmlToBlocks } from './markup.ts';
import { readCoreTitle, type OfficeContext } from './pptx.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader } from './zip.ts';
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
const MAX_WARNINGS = 3;
/**
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
* into a fresh stored zip. Every byte went through the size-limited reader, so
* mammoth's own unzip can never be handed a bomb, and it never spends time
* base64-encoding images we would throw away.
*/
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
const files: Record<string, Uint8Array> = {};
let images = 0;
for (const entry of zip.entries) {
const name = entry.name;
if (name.endsWith('/')) continue;
if (/^word\/media\//i.test(name)) {
images++;
continue;
}
if (!/\.(xml|rels)$/i.test(name)) continue;
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
const data = zip.read(name);
if (data) files[name] = data;
}
return { data: zipSync(files, { level: 0 }), images };
}
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
const seen = new Set<string>();
for (const item of messages) {
const message = String(item?.message ?? '').trim();
// Unknown style names never lose text; they are noise for a reader.
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
}
const unique = [...seen];
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
return out;
}
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
}
const { data, images } = sanitisedPackage(zip);
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const converted = await mammoth.convertToHtml(
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
{
styleMap: STYLE_MAP,
ignoreEmptyParagraphs: true,
// Images are not read locally; never decode them.
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
},
);
const sections: ExtractedSection[] = [];
let heading: string | undefined;
let parts: string[] = [];
const flush = () => {
const text = cleanText(parts.join('\n\n'));
parts = [];
if (!text) return;
const section: ExtractedSection = { text };
if (heading) section.heading = heading;
sections.push(section);
};
for (const block of htmlToBlocks(converted.value)) {
if (block.level >= 1 && block.level <= 3) {
flush();
heading = block.text.slice(0, 200);
}
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
}
flush();
ctx.onProgress?.(1, 1);
const warnings = friendlyWarnings(converted.messages ?? []);
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
let needsOcr = false;
if (totalChars < 25) {
if (images === 0) {
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
}
needsOcr = true;
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
}
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

362
server/documents/extract.ts Normal file
View file

@ -0,0 +1,362 @@
/**
* Document text extraction: kind detection plus one extractor per format.
* Every failure surfaces as an ExtractionError with a user-facing message.
*/
import { cleanText } from './chunk.ts';
import { extractDocx } from './docx.ts';
import { extractPdf } from './pdf.ts';
import { extractPptx } from './pptx.ts';
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
import { listZip, ZipError, type ZipLimits } from './zip.ts';
/** Largest upload accepted (100 MB). */
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
export interface ExtractInput {
bytes: Uint8Array;
filename: string;
mime?: string;
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
onProgress?: (done: number, total: number) => void;
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
signal?: AbortSignal;
}
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
const SNIFF_BYTES = 256 * 1024;
function extensionOf(filename: string): string {
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
const dot = base.lastIndexOf('.');
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
}
function baseMime(mime: string | undefined): string {
return (mime ?? '').split(';')[0].trim().toLowerCase();
}
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
if (bytes.length < offset + signature.length) return false;
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
return true;
}
function isImage(bytes: Uint8Array): boolean {
return (
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
);
}
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
function hasPdfHeader(bytes: Uint8Array): boolean {
const limit = Math.min(bytes.length - 5, 1024);
for (let i = 0; i <= limit; i++) {
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
}
return false;
}
function isZip(bytes: Uint8Array): boolean {
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
}
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
function isOle(bytes: Uint8Array): boolean {
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
}
function isUtf16Bom(bytes: Uint8Array): boolean {
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
}
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
try {
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
return true;
} catch {
return false;
}
}
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
function looksLikeSingleByteText(sample: Uint8Array): boolean {
let high = 0;
for (let i = 0; i < sample.length; i++) {
const b = sample[i];
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
if (b === 0x7f) return false;
if (b >= 0x80) high++;
}
return high <= sample.length * 0.3;
}
function isProbablyText(bytes: Uint8Array): boolean {
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
if (sample.includes(0)) return false;
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
}
/**
* Works out what a file is. Magic bytes win over the name; the extension and
* MIME type are only a fallback. Returns null for anything unsupported
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
*/
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
const ext = extensionOf(filename);
const type = baseMime(mime);
if (!bytes || bytes.length === 0) return null;
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
if (isImage(bytes)) return 'image';
if (isZip(bytes)) {
let names: string[] | null = null;
try {
names = listZip(bytes).map((entry) => entry.name);
} catch {
names = null;
}
if (names) {
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
if (names.includes('ppt/presentation.xml')) return 'pptx';
return null; // xlsx, odt, epub, jar, plain zip...
}
// Unreadable archive: trust the name so extraction can report it as damaged.
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
return null;
}
if (isOle(bytes)) {
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
if (ext === 'docx') return 'docx';
if (ext === 'pptx') return 'pptx';
return null;
}
// No recognisable signature: fall back to the name / declared type.
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
return null;
}
// ---------------------------------------------------------------------------
// Text and Markdown
// ---------------------------------------------------------------------------
function decodeText(bytes: Uint8Array, warnings: string[]): string {
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
return new TextDecoder('utf-8').decode(bytes.subarray(3));
}
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
try {
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch {
if (bytes.includes(0)) {
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
}
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
return new TextDecoder('windows-1252').decode(bytes);
}
}
/** Target size of a plain-text section, in characters. */
const TEXT_SECTION_CHARS = 6000;
function splitPlainText(text: string): ExtractedSection[] {
const sections: ExtractedSection[] = [];
let current: string[] = [];
let size = 0;
const flush = (separator: string) => {
if (current.length === 0) return;
sections.push({ text: current.join(separator) });
current = [];
size = 0;
};
for (const block of text.split(/\n{2,}/)) {
if (!block.trim()) continue;
if (block.length > TEXT_SECTION_CHARS) {
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
flush('\n\n');
for (const line of block.split('\n')) {
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
current.push(line);
size += line.length + 1;
}
flush('\n');
continue;
}
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
current.push(block);
size += block.length + 2;
}
flush('\n\n');
return sections;
}
function markdownHeadingText(raw: string): string {
return raw
.replace(/[ \t]+#+[ \t]*$/, '')
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
.replace(/\*\*|__|[*`]/g, '')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 200);
}
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
const sections: ExtractedSection[] = [];
let title: string | undefined;
let heading: string | undefined;
let lines: string[] = [];
let fence: string | null = null;
const flush = () => {
const body = lines.join('\n').trim();
lines = [];
if (!body) return;
const section: ExtractedSection = { text: body };
if (heading) section.heading = heading;
sections.push(section);
};
for (const line of text.split('\n')) {
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
if (fenceMatch) {
const marker = fenceMatch[1][0];
if (fence === null) fence = marker;
else if (fence === marker) fence = null;
} else if (fence === null && line.length < 400) {
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
if (match) {
const headingText = markdownHeadingText(match[2]);
if (headingText) {
flush();
heading = headingText;
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
}
}
}
lines.push(line);
}
flush();
return { sections, title };
}
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
const warnings: string[] = [];
const text = cleanText(decodeText(bytes, warnings));
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
if (kind === 'markdown') {
const { sections, title } = splitMarkdown(text);
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
if (title) result.title = title;
return result;
}
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
const ext = extensionOf(filename);
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
}
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
}
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
}
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
if (error.code === 'too_large') {
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
}
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
}
/**
* Extracts the text of a document as ordered sections. Images return no
* sections and `needsOcr: true`; the caller runs vision transcription.
*/
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
const bytes = input?.bytes;
const filename = typeof input?.filename === 'string' ? input.filename : '';
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
}
if (bytes.length > MAX_DOCUMENT_BYTES) {
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
}
if (input.signal?.aborted) {
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
}
let kind: DocKind | null;
try {
kind = detectKind(filename, input.mime, bytes);
} catch (error) {
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
}
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
try {
switch (kind) {
case 'pdf':
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
case 'docx':
case 'pptx': {
if (isOle(bytes)) {
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
}
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
}
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
}
case 'text':
case 'markdown': {
const result = extractText(bytes, kind);
input.onProgress?.(1, 1);
return result;
}
case 'image':
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
}
} catch (error) {
if (error instanceof ExtractionError) throw error;
if (error instanceof ZipError) throw fromZipError(error, kind);
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
if (kind === 'docx' || kind === 'pptx') {
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
}
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
}
}

272
server/documents/markup.ts Normal file
View file

@ -0,0 +1,272 @@
/** Small markup helpers shared by the DOCX and PPTX extractors. */
const NAMED_ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
};
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
export function decodeEntities(text: string): string {
if (!text.includes('&')) return text;
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
return String.fromCodePoint(code);
}
return NAMED_ENTITIES[body] ?? match;
});
}
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
export function renderTable(rows: string[][]): string {
const cleaned = rows
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
.filter((row) => row.some((cell) => cell.length > 0));
if (cleaned.length === 0) return '';
const width = Math.max(...cleaned.map((row) => row.length));
const line = (row: string[]) => {
const cells = row.slice();
while (cells.length < width) cells.push('');
return `| ${cells.join(' | ')} |`;
};
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
return out.join('\n');
}
// ---------------------------------------------------------------------------
// HTML (as produced by mammoth) -> Markdown blocks
// ---------------------------------------------------------------------------
export interface MarkdownBlock {
/** 1-6 for headings, 0 for everything else. */
level: number;
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
text: string;
}
interface HtmlElement {
tag: string;
children: HtmlNode[];
}
type HtmlNode = HtmlElement | string;
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
const HEADING_TAG = /^h[1-6]$/;
function parseHtml(html: string): HtmlElement {
const root: HtmlElement = { tag: '#root', children: [] };
const stack: HtmlElement[] = [root];
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
let match: RegExpExecArray | null;
while ((match = tokens.exec(html)) !== null) {
const top = stack[stack.length - 1];
if (match[4] !== undefined) {
top.children.push(decodeEntities(match[4]));
} else if (match[2] === undefined) {
top.children.push('<');
} else {
const tag = match[2].toLowerCase();
if (match[1]) {
// Close the nearest matching open element; ignore stray closers.
for (let i = stack.length - 1; i > 0; i--) {
if (stack[i].tag === tag) {
stack.length = i;
break;
}
}
} else {
const element: HtmlElement = { tag, children: [] };
top.children.push(element);
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
}
}
}
return root;
}
function collapse(text: string): string {
return text.replace(/[ \t\r\n]+/g, ' ');
}
function wrap(marker: string, inner: string): string {
const core = inner.trim();
if (!core) return inner;
const lead = inner.slice(0, inner.length - inner.trimStart().length);
const trail = inner.slice(inner.trimEnd().length);
return `${lead}${marker}${core}${marker}${trail}`;
}
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
function inlineText(nodes: HtmlNode[], plain: boolean): string {
let out = '';
for (const node of nodes) {
if (typeof node === 'string') {
out += collapse(node);
continue;
}
switch (node.tag) {
case 'br':
out += '\n';
break;
case 'img':
case 'hr':
break;
case 'strong':
case 'b':
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
break;
case 'em':
case 'i':
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
break;
case 'a': {
const text = inlineText(node.children, plain);
// mammoth's footnote back-references
if (text.trim() !== '↑') out += text;
break;
}
case 'p':
case 'li':
case 'tr':
case 'td':
case 'th':
case 'ul':
case 'ol':
case 'table': {
const inner = inlineText(node.children, plain).trim();
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
break;
}
default:
out += inlineText(node.children, plain);
}
}
return out;
}
function tidyInline(text: string): string {
return text
.split('\n')
.map((line) => line.replace(/ {2,}/g, ' ').trim())
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
.join('\n')
.trim();
}
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
const ordered = list.tag === 'ol';
let counter = 0;
for (const item of list.children) {
if (typeof item === 'string' || item.tag !== 'li') continue;
counter++;
const marker = ordered ? `${counter}. ` : '- ';
const inlineNodes: HtmlNode[] = [];
const nested: HtmlElement[] = [];
const paragraphs: string[] = [];
const flushInline = () => {
const text = tidyInline(inlineText(inlineNodes, false));
if (text) paragraphs.push(text);
inlineNodes.length = 0;
};
for (const child of item.children) {
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
flushInline();
nested.push(child);
} else if (typeof child !== 'string' && child.tag === 'p') {
flushInline();
inlineNodes.push(...child.children);
flushInline();
} else if (typeof child !== 'string' && child.tag === 'table') {
flushInline();
const table = renderHtmlTable(child);
if (table) paragraphs.push(table);
} else {
inlineNodes.push(child);
}
}
flushInline();
const pad = indent + ' '.repeat(marker.length);
const body = paragraphs.join('\n').split('\n');
if (body.length > 0 && body[0] !== '') {
lines.push(indent + marker + body[0]);
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
} else if (nested.length === 0) {
continue;
}
for (const child of nested) renderList(child, pad, lines);
}
}
function renderHtmlTable(table: HtmlElement): string {
const rows: string[][] = [];
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') continue;
if (node.tag === 'tr') {
const cells: string[] = [];
for (const cell of node.children) {
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
cells.push(tidyInline(inlineText(cell.children, false)));
}
rows.push(cells);
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
visit(node.children);
}
}
};
visit(table.children);
return renderTable(rows);
}
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
export function htmlToBlocks(html: string): MarkdownBlock[] {
const root = parseHtml(html);
const blocks: MarkdownBlock[] = [];
const loose: HtmlNode[] = [];
const flushLoose = () => {
if (loose.length === 0) return;
const text = tidyInline(inlineText(loose, false));
if (text) blocks.push({ level: 0, text });
loose.length = 0;
};
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') {
loose.push(node);
continue;
}
if (HEADING_TAG.test(node.tag)) {
flushLoose();
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
if (text) blocks.push({ level: Number(node.tag[1]), text });
} else if (node.tag === 'p') {
flushLoose();
const text = tidyInline(inlineText(node.children, false));
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'ul' || node.tag === 'ol') {
flushLoose();
const lines: string[] = [];
renderList(node, '', lines);
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
} else if (node.tag === 'table') {
flushLoose();
const text = renderHtmlTable(node);
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
flushLoose();
visit(node.children);
} else {
loose.push(node);
}
}
};
visit(root.children);
flushLoose();
return blocks;
}

249
server/documents/pdf.ts Normal file
View file

@ -0,0 +1,249 @@
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
import { getDocumentProxy } from 'unpdf';
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
export interface PdfContext {
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
/** Below this average of characters per page the PDF is treated as a scan. */
const OCR_CHARS_PER_PAGE = 25;
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
const CLEANUP_EVERY = 40;
interface TextItemLike {
str?: unknown;
hasEOL?: unknown;
transform?: unknown;
height?: unknown;
}
interface Line {
text: string;
y: number | null;
height: number;
}
/**
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
* order (which keeps multi-column pages readable) with `hasEOL` marking line
* ends; the vertical gaps between lines are then used to restore paragraph
* breaks, which pdf.js does not report.
*/
export function textItemsToText(items: readonly TextItemLike[]): string {
const lines: Line[] = [];
let text = '';
let y: number | null = null;
let height = 0;
const endLine = () => {
lines.push({ text, y, height });
text = '';
y = null;
height = 0;
};
for (const item of items) {
if (!item || typeof item.str !== 'string') continue;
const str = item.str;
if (str.length > 0) {
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
const t = item.transform as number[];
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
if (upright && Number.isFinite(t[5])) {
y = t[5];
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
height = Number.isFinite(h) ? h : 0;
}
}
text += str;
}
if (item.hasEOL === true) endLine();
}
if (text.length > 0) endLine();
// Typical line advance on this page (median of the downward steps).
const advances: number[] = [];
for (let i = 1; i < lines.length; i++) {
const a = lines[i - 1];
const b = lines[i];
if (a.y === null || b.y === null) continue;
const dy = a.y - b.y;
const ref = Math.max(a.height, b.height, 1);
if (dy > 0 && dy < ref * 4) advances.push(dy);
}
advances.sort((p, q) => p - q);
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
let out = '';
let prev: Line | null = null;
for (const line of lines) {
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
if (!content) {
// An empty line is an explicit gap.
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
continue;
}
if (out && !out.endsWith('\n')) {
let gap = false;
if (prev && prev.y !== null && line.y !== null) {
const dy = prev.y - line.y;
const ref = Math.max(prev.height, line.height, 1);
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
else gap = dy > ref * 1.9;
}
out += gap ? '\n\n' : '\n';
}
out += content;
prev = line;
}
return out;
}
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
export function tidyTitle(raw: unknown): string | undefined {
if (typeof raw !== 'string') return undefined;
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
return title;
}
function errorName(error: unknown): string {
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
}
function toExtractionError(error: unknown): ExtractionError {
if (error instanceof ExtractionError) return error;
const name = errorName(error);
const message = error instanceof Error ? error.message : String(error);
if (name === 'PasswordException' || /password/i.test(message)) {
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
}
if (
name === 'InvalidPDFException' ||
name === 'FormatError' ||
name === 'XRefParseException' ||
name === 'XRefEntryException' ||
name === 'MissingDataException' ||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
) {
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
}
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
}
function cancelled(signal: AbortSignal): ExtractionError {
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
}
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
const { onProgress, signal } = ctx;
if (signal?.aborted) throw cancelled(signal);
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
try {
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
pdf = await getDocumentProxy(bytes.slice(), options);
} catch (error) {
throw toExtractionError(error);
}
try {
const total = pdf.numPages;
if (!Number.isFinite(total) || total < 1) {
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
}
let title: string | undefined;
try {
const meta = await pdf.getMetadata();
const info = (meta?.info ?? {}) as Record<string, unknown>;
title = tidyTitle(info.Title);
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
} catch {
// Metadata is optional.
}
const pageTexts: string[] = new Array(total).fill('');
let failedPages = 0;
let lastYield = Date.now();
for (let n = 1; n <= total; n++) {
if (signal?.aborted) throw cancelled(signal);
try {
const page = await pdf.getPage(n);
try {
const content = await page.getTextContent();
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
} finally {
page.cleanup();
}
} catch (error) {
if (errorName(error) === 'PasswordException') throw error;
failedPages++;
}
if (n % CLEANUP_EVERY === 0) {
try {
await pdf.cleanup();
} catch {
// Best effort.
}
}
onProgress?.(n, total);
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
if (failedPages === total) {
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
}
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
const sections: ExtractedSection[] = [];
let empty = 0;
let chars = 0;
for (let i = 0; i < total; i++) {
const text = cleaned[i];
if (!text) {
empty++;
continue;
}
chars += text.length;
sections.push({ text, page: i + 1 });
}
const warnings: string[] = [];
if (empty > 0) {
warnings.push(
empty === total
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
: `${empty} of ${total} pages had no extractable text.`,
);
}
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
const result: ExtractResult = {
kind: 'pdf',
sections,
pageCount: total,
warnings,
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
};
if (title) result.title = title;
return result;
} catch (error) {
throw toExtractionError(error);
} finally {
try {
await pdf.loadingTask.destroy();
} catch {
// Nothing useful to do.
}
}
}

424
server/documents/pptx.ts Normal file
View file

@ -0,0 +1,424 @@
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
import { XMLParser } from 'fast-xml-parser';
import { cleanText } from './chunk.ts';
import { decodeEntities, renderTable } from './markup.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader, type ZipLimits } from './zip.ts';
export interface OfficeContext {
limits: ZipLimits;
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
type XmlNode = Record<string, unknown>;
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '',
preserveOrder: true,
trimValues: false,
parseTagValue: false,
parseAttributeValue: false,
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
processEntities: false,
ignoreDeclaration: true,
ignorePiTags: true,
});
function parseXml(xml: string): XmlNode[] {
const parsed = parser.parse(xml) as unknown;
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
}
function tagOf(node: XmlNode): string {
for (const key in node) if (key !== ':@') return key;
return '';
}
/** Element name without its namespace prefix. */
function localName(node: XmlNode): string {
const tag = tagOf(node);
const colon = tag.indexOf(':');
return colon < 0 ? tag : tag.slice(colon + 1);
}
function childrenOf(node: XmlNode): XmlNode[] {
const value = node[tagOf(node)];
return Array.isArray(value) ? (value as XmlNode[]) : [];
}
function attr(node: XmlNode, name: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
const direct = attrs[name];
if (typeof direct === 'string') return direct;
return undefined;
}
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
function attrLocal(node: XmlNode, local: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
for (const key in attrs) {
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
}
return undefined;
}
function child(node: XmlNode, local: string): XmlNode | undefined {
for (const c of childrenOf(node)) if (localName(c) === local) return c;
return undefined;
}
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
if (depth > 40) return undefined;
for (const node of nodes) {
if (localName(node) === local) return node;
const found = findDeep(childrenOf(node), local, depth + 1);
if (found) return found;
}
return undefined;
}
/** Concatenated text of every a:t / m:t descendant. */
function deepText(node: XmlNode, depth = 0): string {
if (depth > 40) return '';
const name = localName(node);
if (name === '#text') return '';
if (name === 't') {
let out = '';
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
return decodeEntities(out);
}
let out = '';
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
return out;
}
interface Paragraph {
text: string;
level: number;
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
bullet: boolean | undefined;
}
function readParagraph(p: XmlNode): Paragraph {
let text = '';
let level = 0;
let bullet: boolean | undefined;
for (const c of childrenOf(p)) {
switch (localName(c)) {
case 'pPr': {
const lvl = Number(attr(c, 'lvl'));
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
for (const b of childrenOf(c)) {
const name = localName(b);
if (name === 'buNone') bullet = false;
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
}
break;
}
case 'br':
text += '\n';
break;
case 'fld': {
const type = attr(c, 'type') ?? '';
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
break;
}
case 'endParaRPr':
case '#text':
break;
default:
text += deepText(c);
}
}
text = text
.replace(/[\u000B\u2028]/g, '\n')
.split('\n')
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
.filter((line) => line.length > 0)
.join('\n');
return { text, level, bullet };
}
function readParagraphs(txBody: XmlNode): Paragraph[] {
const out: Paragraph[] = [];
for (const c of childrenOf(txBody)) {
if (localName(c) !== 'p') continue;
const paragraph = readParagraph(c);
if (paragraph.text) out.push(paragraph);
}
return out;
}
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
const lines: string[] = [];
for (const p of paragraphs) {
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
if (!bulleted) {
lines.push(p.text);
continue;
}
const indent = ' '.repeat(p.level);
const parts = p.text.split('\n');
lines.push(`${indent}- ${parts[0]}`);
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
}
return lines.join('\n');
}
function readTable(tbl: XmlNode): string {
const rows: string[][] = [];
for (const tr of childrenOf(tbl)) {
if (localName(tr) !== 'tr') continue;
const cells: string[] = [];
for (const tc of childrenOf(tr)) {
if (localName(tc) !== 'tc') continue;
const txBody = child(tc, 'txBody');
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
}
rows.push(cells);
}
return renderTable(rows);
}
interface SlideContent {
title: string;
blocks: string[];
}
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
function placeholderType(shape: XmlNode): string | null {
const nv = child(shape, 'nvSpPr');
const nvPr = nv ? child(nv, 'nvPr') : undefined;
const ph = nvPr ? child(nvPr, 'ph') : undefined;
if (!ph) return null;
return attr(ph, 'type') ?? 'body';
}
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
if (depth > 30) return;
for (const node of nodes) {
switch (localName(node)) {
case 'sp': {
const type = placeholderType(node);
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
const txBody = child(node, 'txBody');
if (!txBody) break;
const paragraphs = readParagraphs(txBody);
if (paragraphs.length === 0) break;
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
} else {
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
const text = renderParagraphs(paragraphs, bullets);
if (text) into.blocks.push(text);
}
break;
}
case 'grpSp':
walkShapes(childrenOf(node), into, depth + 1);
break;
case 'graphicFrame': {
const tbl = findDeep(childrenOf(node), 'tbl');
if (tbl) {
const table = readTable(tbl);
if (table) into.blocks.push(table);
}
break;
}
case 'AlternateContent': {
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
break;
}
default:
break;
}
}
}
function readSlide(xml: string): SlideContent {
const content: SlideContent = { title: '', blocks: [] };
const spTree = findDeep(parseXml(xml), 'spTree');
if (spTree) walkShapes(childrenOf(spTree), content, 0);
return content;
}
function readNotes(xml: string): string {
const spTree = findDeep(parseXml(xml), 'spTree');
if (!spTree) return '';
const parts: string[] = [];
for (const shape of childrenOf(spTree)) {
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
const txBody = child(shape, 'txBody');
if (!txBody) continue;
const text = renderParagraphs(readParagraphs(txBody), false);
if (text) parts.push(text);
}
return parts.join('\n');
}
interface Relationship {
id: string;
type: string;
target: string;
external: boolean;
}
function readRelationships(xml: string | null): Relationship[] {
if (!xml) return [];
const out: Relationship[] = [];
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
let match: RegExpExecArray | null;
while ((match = tags.exec(xml)) !== null) {
const attrs: Record<string, string> = {};
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
let pair: RegExpExecArray | null;
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
if (attrs.Id && attrs.Target) {
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
}
}
return out;
}
/** Resolves a relationship target against the directory of the part that owns it. */
function resolveTarget(baseDir: string, target: string): string {
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
for (const segment of target.split('/')) {
if (!segment || segment === '.') continue;
if (segment === '..') parts.pop();
else parts.push(segment);
}
return parts.join('/');
}
function relsPathFor(part: string): string {
const slash = part.lastIndexOf('/');
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
}
function slideOrder(zip: ZipReader): string[] {
const ordered: string[] = [];
const presentation = zip.readText('ppt/presentation.xml');
if (presentation) {
const targets = new Map<string, string>();
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
}
const list = findDeep(parseXml(presentation), 'sldIdLst');
if (list) {
for (const item of childrenOf(list)) {
if (localName(item) !== 'sldId') continue;
const id = attrLocal(item, 'id');
const path = id ? targets.get(id) : undefined;
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
}
}
}
if (ordered.length > 0) return ordered;
// Damaged or unusual package: fall back to numeric file order.
return zip
.names()
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
.map((item) => item.name);
}
export function readCoreTitle(zip: ZipReader): string | undefined {
let xml: string | null = null;
try {
xml = zip.readText('docProps/core.xml');
} catch {
return undefined;
}
if (!xml) return undefined;
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
return title || undefined;
}
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
const slides = slideOrder(zip);
if (slides.length === 0) {
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
}
const sections: ExtractedSection[] = [];
const warnings: string[] = [];
let withoutText = 0;
let unreadable = 0;
let totalChars = 0;
let lastYield = Date.now();
for (let i = 0; i < slides.length; i++) {
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const path = slides[i];
let text = '';
let title = '';
try {
const xml = zip.readText(path);
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
title = slide.title;
const parts: string[] = [];
if (slide.title) parts.push(`## ${slide.title}`);
parts.push(...slide.blocks);
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
);
if (notesRel) {
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
const notes = notesXml ? readNotes(notesXml) : '';
if (notes) parts.push(`Notes:\n${notes}`);
}
text = cleanText(parts.join('\n\n'));
} catch (error) {
if (error instanceof ExtractionError) throw error;
if ((error as { code?: string })?.code === 'too_large') throw error;
unreadable++;
}
if (text) {
totalChars += text.length;
const section: ExtractedSection = { text, page: i + 1 };
if (title) section.heading = title;
sections.push(section);
} else {
withoutText++;
}
ctx.onProgress?.(i + 1, slides.length);
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
const total = slides.length;
if (unreadable === total) {
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
}
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
if (withoutText === total) {
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
} else if (withoutText * 2 > total) {
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
}
const result: ExtractResult = {
kind: 'pptx',
sections,
pageCount: total,
warnings,
needsOcr: totalChars < total * 25,
};
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}

View file

@ -0,0 +1,270 @@
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
import { strToU8, zipSync } from 'fflate';
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
function pdfString(text: string): string {
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
}
export interface PdfPage {
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
lines?: string[];
/** Raw content stream, used instead of `lines`. */
raw?: string;
}
export interface PdfOptions {
title?: string;
/** Adds a standard security handler whose user password is not empty. */
encrypted?: boolean;
}
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
const objects: string[] = [];
const add = (body: string): number => {
objects.push(body);
return objects.length;
};
const catalogId = add('');
const pagesId = add('');
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
const pageIds: number[] = [];
for (const page of pages) {
let stream = page.raw ?? '';
if (page.raw === undefined && page.lines && page.lines.length > 0) {
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
for (const line of page.lines) {
if (line === '') ops.push('0 -14 Td');
else ops.push(`${pdfString(line)} Tj`, 'T*');
}
ops.push('ET');
stream = ops.join('\n');
}
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
pageIds.push(
add(
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
),
);
}
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
let infoId = 0;
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
let encryptId = 0;
if (options.encrypted) {
const o = 'a1'.repeat(32);
const u = 'b2'.repeat(32);
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
}
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
const offsets: number[] = [];
objects.forEach((body, index) => {
offsets.push(out.length);
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
});
const xrefOffset = out.length;
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
if (infoId) out += ` /Info ${infoId} 0 R`;
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
return latin1(out);
}
const xmlEscape = (text: string): string => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
function coreProps(title: string | undefined): string {
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
}
// ---------------------------------------------------------------------------
// DOCX
// ---------------------------------------------------------------------------
export type DocxBlock =
| { type: 'heading'; level: number; text: string }
| { type: 'p'; text: string; bold?: string }
| { type: 'bullet'; text: string; level?: number }
| { type: 'number'; text: string; level?: number }
| { type: 'table'; rows: string[][] }
| { type: 'image' };
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
function docxRun(text: string, bold = false): string {
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
}
function docxBlock(block: DocxBlock): string {
switch (block.type) {
case 'heading':
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
case 'p':
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
case 'bullet':
case 'number':
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
case 'table':
return `<w:tbl>${block.rows
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
.join('')}</w:tbl>`;
case 'image':
return '<w:p/>';
}
}
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
const levels = (format: string) =>
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
),
'word/_rels/document.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
),
'word/numbering.xml': strToU8(
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
),
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
'docProps/core.xml': strToU8(coreProps(options.title)),
...options.extraFiles,
};
return zipSync(files);
}
// ---------------------------------------------------------------------------
// PPTX
// ---------------------------------------------------------------------------
export interface PptxSlide {
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
file: number;
title?: string;
/** Body placeholder paragraphs. */
bullets?: Array<string | { text: string; level: number }>;
/** A free text box (not a placeholder). */
textBox?: string[];
table?: string[][];
notes?: string[];
}
const P_NS =
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
function aParagraph(text: string, level = 0): string {
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
}
function shape(id: number, placeholder: string | null, paragraphs: string): string {
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
}
function slideXml(slide: PptxSlide): string {
const shapes: string[] = [];
let id = 2;
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
if (slide.bullets) {
shapes.push(
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
);
}
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
if (slide.table) {
const rows = slide.table
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
.join('');
shapes.push(
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
);
}
// Every slide also carries a slide-number placeholder, which must be ignored.
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
}
function notesXml(lines: string[]): string {
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
2,
'sldImg',
'',
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
}
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
const files: Record<string, Uint8Array> = {
'[Content_Types].xml': strToU8(
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
),
'_rels/.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
),
'docProps/core.xml': strToU8(coreProps(options.title)),
'ppt/presentation.xml': strToU8(
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
.join('')}</p:sldIdLst></p:presentation>`,
),
'ppt/_rels/presentation.xml.rels': strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
.join('')}</Relationships>`,
),
};
for (const slide of slides) {
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
if (slide.notes) {
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
);
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
}
}
return zipSync(files);
}
// ---------------------------------------------------------------------------
// Misc
// ---------------------------------------------------------------------------
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
export const TINY_PNG = Uint8Array.from([
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
]);
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
for (let i = 0; i + 46 < zip.length; i++) {
if (view.getUint32(i, true) !== 0x02014b50) continue;
const nameLen = view.getUint16(i + 28, true);
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
if (entryName === name) view.setUint32(i + 24, 16, true);
}
return zip;
}

53
server/documents/types.ts Normal file
View file

@ -0,0 +1,53 @@
/** Contract for the document extraction + chunking pipeline. */
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
export interface ExtractedSection {
/** Plain text (Markdown allowed) for this page / slide / section. */
text: string;
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
page?: number;
/** Nearest heading or slide title, when known. */
heading?: string;
}
export interface ExtractResult {
kind: DocKind;
sections: ExtractedSection[];
/** Page or slide count for paged formats. */
pageCount?: number;
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
warnings: string[];
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
needsOcr: boolean;
/** Document title from metadata, when present. */
title?: string;
}
export interface DocChunk {
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
}
export interface ChunkOptions {
/** Preferred chunk size in estimated tokens. */
targetTokens?: number;
/** Hard ceiling; a chunk is split before exceeding it. */
maxTokens?: number;
/** Tokens of trailing context repeated at the start of the next chunk. */
overlapTokens?: number;
}
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
export class ExtractionError extends Error {
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
super(message, opts);
this.name = 'ExtractionError';
this.code = code;
}
}

213
server/documents/zip.ts Normal file
View file

@ -0,0 +1,213 @@
/**
* Minimal, defensive ZIP reader for OOXML packages (DOCX / PPTX).
*
* Reads the central directory itself and inflates entries through fflate's
* streaming inflater in small input slices, so both memory and CPU are bounded
* by the limits below even when the archive lies about its sizes (zip bombs).
*/
import { Inflate } from 'fflate';
export interface ZipEntry {
name: string;
method: number;
compressedSize: number;
size: number;
offset: number;
encrypted: boolean;
}
export interface ZipLimits {
/** Largest uncompressed size accepted for a single entry. */
maxEntryBytes: number;
/** Largest total of uncompressed bytes read through one reader. */
maxTotalBytes: number;
}
export class ZipError extends Error {
code: 'corrupt' | 'too_large';
constructor(code: ZipError['code'], message: string) {
super(message);
this.name = 'ZipError';
this.code = code;
}
}
const EOCD_SIG = 0x06054b50;
const ZIP64_LOCATOR_SIG = 0x07064b50;
const ZIP64_EOCD_SIG = 0x06064b50;
const CENTRAL_SIG = 0x02014b50;
const LOCAL_SIG = 0x04034b50;
const MAX_ENTRIES = 20_000;
const INFLATE_STEP = 64 * 1024;
const nameDecoder = new TextDecoder('utf-8');
function u16(b: Uint8Array, o: number): number {
return b[o] | (b[o + 1] << 8);
}
function u32(b: Uint8Array, o: number): number {
return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
}
function u64(b: Uint8Array, o: number): number {
return u32(b, o) + u32(b, o + 4) * 0x1_0000_0000;
}
/** Parses the central directory. Throws ZipError('corrupt') when it cannot be read. */
export function listZip(bytes: Uint8Array): ZipEntry[] {
const len = bytes.length;
if (len < 22) throw new ZipError('corrupt', 'Not a zip archive');
let eocd = -1;
const stop = Math.max(0, len - 22 - 0xffff);
for (let i = len - 22; i >= stop; i--) {
if (bytes[i] === 0x50 && u32(bytes, i) === EOCD_SIG) {
eocd = i;
break;
}
}
if (eocd < 0) throw new ZipError('corrupt', 'Zip directory not found');
let count = u16(bytes, eocd + 10);
let dirOffset = u32(bytes, eocd + 16);
if (count === 0xffff || dirOffset === 0xffffffff) {
const locator = eocd - 20;
if (locator < 0 || u32(bytes, locator) !== ZIP64_LOCATOR_SIG) throw new ZipError('corrupt', 'Bad zip64 locator');
const z = u64(bytes, locator + 8);
if (z + 56 > len || u32(bytes, z) !== ZIP64_EOCD_SIG) throw new ZipError('corrupt', 'Bad zip64 directory');
count = u64(bytes, z + 32);
dirOffset = u64(bytes, z + 48);
}
if (count > MAX_ENTRIES) throw new ZipError('too_large', 'Too many files in archive');
const entries: ZipEntry[] = [];
let p = dirOffset;
for (let i = 0; i < count; i++) {
if (p + 46 > len || u32(bytes, p) !== CENTRAL_SIG) throw new ZipError('corrupt', 'Bad zip directory entry');
const flags = u16(bytes, p + 8);
const method = u16(bytes, p + 10);
let compressedSize = u32(bytes, p + 20);
let size = u32(bytes, p + 24);
const nameLen = u16(bytes, p + 28);
const extraLen = u16(bytes, p + 30);
const commentLen = u16(bytes, p + 32);
let offset = u32(bytes, p + 42);
const nameStart = p + 46;
const extraStart = nameStart + nameLen;
const next = extraStart + extraLen + commentLen;
if (next > len) throw new ZipError('corrupt', 'Truncated zip directory');
if (size === 0xffffffff || compressedSize === 0xffffffff || offset === 0xffffffff) {
// zip64 extended information: only the saturated fields are present, in this order.
let e = extraStart;
const extraEnd = extraStart + extraLen;
while (e + 4 <= extraEnd) {
const id = u16(bytes, e);
const dataLen = u16(bytes, e + 2);
if (id === 1) {
let q = e + 4;
if (size === 0xffffffff && q + 8 <= extraEnd) (size = u64(bytes, q)), (q += 8);
if (compressedSize === 0xffffffff && q + 8 <= extraEnd) (compressedSize = u64(bytes, q)), (q += 8);
if (offset === 0xffffffff && q + 8 <= extraEnd) offset = u64(bytes, q);
break;
}
e += 4 + dataLen;
}
}
const name = nameDecoder.decode(bytes.subarray(nameStart, extraStart)).replace(/\\/g, '/');
entries.push({ name, method, compressedSize, size, offset, encrypted: (flags & 1) === 1 });
p = next;
}
return entries;
}
/** Random-access reader over a zip held in memory, with per-entry and total size limits. */
export class ZipReader {
private readonly bytes: Uint8Array;
private readonly limits: ZipLimits;
private readonly byName = new Map<string, ZipEntry>();
private totalRead = 0;
readonly entries: ZipEntry[];
constructor(bytes: Uint8Array, limits: ZipLimits) {
this.bytes = bytes;
this.limits = limits;
this.entries = listZip(bytes);
for (const entry of this.entries) if (!this.byName.has(entry.name)) this.byName.set(entry.name, entry);
}
has(name: string): boolean {
return this.byName.has(name);
}
names(): string[] {
return this.entries.map((entry) => entry.name);
}
/** Uncompressed bytes of an entry, or null when it does not exist. */
read(name: string): Uint8Array | null {
const entry = this.byName.get(name);
if (!entry) return null;
if (entry.encrypted) throw new ZipError('corrupt', 'Encrypted zip entry');
if (entry.size > this.limits.maxEntryBytes) throw new ZipError('too_large', `Entry too large: ${name}`);
if (this.totalRead + entry.size > this.limits.maxTotalBytes) throw new ZipError('too_large', 'Archive too large');
const b = this.bytes;
const o = entry.offset;
if (o + 30 > b.length || u32(b, o) !== LOCAL_SIG) throw new ZipError('corrupt', 'Bad zip file header');
const dataStart = o + 30 + u16(b, o + 26) + u16(b, o + 28);
const dataEnd = dataStart + entry.compressedSize;
if (dataEnd > b.length) throw new ZipError('corrupt', 'Truncated zip entry');
const compressed = b.subarray(dataStart, dataEnd);
let out: Uint8Array;
if (entry.method === 0) {
if (compressed.length !== entry.size) throw new ZipError('corrupt', 'Zip entry size mismatch');
out = compressed;
} else if (entry.method === 8) {
out = this.inflate(compressed, entry.size);
} else {
throw new ZipError('corrupt', `Unsupported zip compression method ${entry.method}`);
}
this.totalRead += out.length;
return out;
}
readText(name: string): string | null {
const data = this.read(name);
if (!data) return null;
let text = new TextDecoder('utf-8').decode(data);
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
return text;
}
private inflate(compressed: Uint8Array, expected: number): Uint8Array {
const out = new Uint8Array(expected);
let written = 0;
let overflow = false;
const inflater = new Inflate((chunk) => {
if (overflow) return;
if (written + chunk.length > expected) {
overflow = true;
return;
}
out.set(chunk, written);
written += chunk.length;
});
try {
if (compressed.length === 0) {
inflater.push(compressed, true);
} else {
for (let p = 0; p < compressed.length && !overflow; p += INFLATE_STEP) {
const end = Math.min(p + INFLATE_STEP, compressed.length);
inflater.push(compressed.subarray(p, end), end === compressed.length);
}
}
} catch {
throw new ZipError('corrupt', 'Zip entry could not be decompressed');
}
// More data than the directory declared is the signature of a zip bomb.
if (overflow) throw new ZipError('too_large', 'Zip entry larger than declared');
if (written !== expected) throw new ZipError('corrupt', 'Zip entry size mismatch');
return out;
}
}