Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
514
server/documents/chunk.ts
Normal file
514
server/documents/chunk.ts
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
/**
|
||||
* Text normalisation, token estimation and retrieval chunking.
|
||||
* Pure functions, no dependencies. Everything here is linear-time on its input.
|
||||
*/
|
||||
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Token estimation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Un-rounded token estimate. Sub-additive under concatenation
|
||||
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
|
||||
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
|
||||
*/
|
||||
function rawTokens(text: string): number {
|
||||
let total = 0;
|
||||
let prevWhitespace = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text.charCodeAt(i);
|
||||
if (c === 32 || (c >= 9 && c <= 13)) {
|
||||
// A run of whitespace costs the same as one character of prose.
|
||||
if (!prevWhitespace) total += 0.25;
|
||||
prevWhitespace = true;
|
||||
continue;
|
||||
}
|
||||
prevWhitespace = false;
|
||||
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
|
||||
else if (c >= 48 && c <= 57) total += 0.4;
|
||||
else if (c < 128) total += 0.5;
|
||||
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
|
||||
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
|
||||
else total += 0.5; // accented Latin, Greek, symbols
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
|
||||
export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
return Math.ceil(rawTokens(text));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// cleanText
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LIGATURES: Record<string, string> = {
|
||||
'ff': 'ff',
|
||||
'fi': 'fi',
|
||||
'fl': 'fl',
|
||||
'ffi': 'ffi',
|
||||
'ffl': 'ffl',
|
||||
'ſt': 'st',
|
||||
'st': 'st',
|
||||
};
|
||||
|
||||
/** First halves that normally form a real hyphenated compound ("well-known"). */
|
||||
const HYPHEN_KEEP_LEFT = new Set([
|
||||
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
|
||||
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
|
||||
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
|
||||
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
|
||||
]);
|
||||
|
||||
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
|
||||
const HYPHEN_KEEP_RIGHT = new Set([
|
||||
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
|
||||
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
|
||||
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
|
||||
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
|
||||
]);
|
||||
|
||||
function isLower(code: number): boolean {
|
||||
return code >= 97 && code <= 122;
|
||||
}
|
||||
|
||||
function isLetter(code: number): boolean {
|
||||
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
|
||||
}
|
||||
|
||||
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
|
||||
function joinHyphenatedLineBreaks(text: string): string {
|
||||
if (!text.includes('-\n')) return text;
|
||||
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
|
||||
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
|
||||
let start = offset;
|
||||
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
|
||||
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
|
||||
const left = whole.slice(start, offset);
|
||||
if (left.length < 2) return match;
|
||||
if (start > 0) {
|
||||
const before = whole.charCodeAt(start - 1);
|
||||
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
|
||||
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
|
||||
}
|
||||
const rightStart = offset + match.length;
|
||||
let rightEnd = rightStart;
|
||||
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
|
||||
const right = whole.slice(rightStart, rightEnd);
|
||||
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
|
||||
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
|
||||
return '';
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalises extracted text: newlines, unicode spaces, control characters,
|
||||
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
|
||||
* a line break. Idempotent.
|
||||
*/
|
||||
export function cleanText(text: string): string {
|
||||
if (!text) return '';
|
||||
let t = text;
|
||||
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
|
||||
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
|
||||
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
|
||||
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
|
||||
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
|
||||
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
|
||||
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
|
||||
t = t
|
||||
.split('\n')
|
||||
.map((line) => line.trimEnd())
|
||||
.join('\n');
|
||||
t = joinHyphenatedLineBreaks(t);
|
||||
t = t.replace(/\n{3,}/g, '\n\n');
|
||||
return t.trim();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Repeated page headers / footers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function edgeLineKey(line: string): string | null {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.length > 100) return null;
|
||||
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes running headers/footers and page numbers from per-page text, but only
|
||||
* when the same first or last line (digits ignored) shows up on at least 60% of
|
||||
* the pages that have text, and there are at least four such pages. Chapter
|
||||
* headings that change through a book are therefore left alone.
|
||||
*/
|
||||
export function stripRepeatedPageLines(pages: string[]): string[] {
|
||||
let current = pages.map((page) => page.split('\n'));
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
const firstIdx: number[] = [];
|
||||
const lastIdx: number[] = [];
|
||||
const topCounts = new Map<string, number>();
|
||||
const bottomCounts = new Map<string, number>();
|
||||
let textPages = 0;
|
||||
for (let p = 0; p < current.length; p++) {
|
||||
const lines = current[p];
|
||||
let first = 0;
|
||||
while (first < lines.length && !lines[first].trim()) first++;
|
||||
let last = lines.length - 1;
|
||||
while (last >= 0 && !lines[last].trim()) last--;
|
||||
firstIdx.push(first < lines.length ? first : -1);
|
||||
lastIdx.push(last);
|
||||
if (last < 0) continue;
|
||||
textPages++;
|
||||
const top = edgeLineKey(lines[first]);
|
||||
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
|
||||
if (last !== first) {
|
||||
const bottom = edgeLineKey(lines[last]);
|
||||
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
if (textPages < 4) break;
|
||||
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
|
||||
let removed = false;
|
||||
current = current.map((lines, p) => {
|
||||
const first = firstIdx[p];
|
||||
const last = lastIdx[p];
|
||||
if (last < 0) return lines;
|
||||
let from = 0;
|
||||
let to = lines.length;
|
||||
const top = edgeLineKey(lines[first]);
|
||||
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
|
||||
if (last !== first) {
|
||||
const bottom = edgeLineKey(lines[last]);
|
||||
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
|
||||
}
|
||||
if (from === 0 && to === lines.length) return lines;
|
||||
removed = true;
|
||||
return lines.slice(from, to);
|
||||
});
|
||||
if (!removed) break;
|
||||
}
|
||||
return current.map((lines) => lines.join('\n'));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
|
||||
interface Atom {
|
||||
text: string;
|
||||
raw: number;
|
||||
sep: string;
|
||||
sepRaw: number;
|
||||
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
|
||||
strength: number;
|
||||
}
|
||||
|
||||
const WHITESPACE_RAW = 0.25;
|
||||
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
|
||||
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
|
||||
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
|
||||
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
|
||||
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
|
||||
|
||||
function sepRawOf(sep: string): number {
|
||||
return sep ? WHITESPACE_RAW : 0;
|
||||
}
|
||||
|
||||
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
|
||||
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
|
||||
let start = 0;
|
||||
let acc = 0;
|
||||
let first = true;
|
||||
const push = (end: number) => {
|
||||
const text = word.slice(start, end);
|
||||
out.push({
|
||||
text,
|
||||
raw: rawTokens(text),
|
||||
sep: first ? sep : '',
|
||||
sepRaw: first ? sepRawOf(sep) : 0,
|
||||
strength: first ? strength : -1,
|
||||
});
|
||||
first = false;
|
||||
start = end;
|
||||
acc = 0;
|
||||
};
|
||||
for (let i = 0; i < word.length; i++) {
|
||||
const c = word.charCodeAt(i);
|
||||
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
|
||||
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
|
||||
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
|
||||
acc += weight;
|
||||
}
|
||||
if (start < word.length) push(word.length);
|
||||
}
|
||||
|
||||
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
|
||||
const raw = rawTokens(sentence);
|
||||
if (raw <= atomMax) {
|
||||
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
|
||||
return;
|
||||
}
|
||||
// Oversized sentence (or table row, or blob): fall back to groups of words.
|
||||
const groupMax = Math.max(1, Math.min(atomMax, 24));
|
||||
const words = sentence.split(/[ \t]+/);
|
||||
// Keep leading indentation attached to the first word.
|
||||
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
|
||||
let group = '';
|
||||
let groupRaw = 0;
|
||||
let first = true;
|
||||
const flush = () => {
|
||||
if (!group) return;
|
||||
out.push({
|
||||
text: group,
|
||||
raw: rawTokens(group),
|
||||
sep: first ? sep : ' ',
|
||||
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
|
||||
strength: first ? strength : 0,
|
||||
});
|
||||
first = false;
|
||||
group = '';
|
||||
groupRaw = 0;
|
||||
};
|
||||
let pendingIndent = indent;
|
||||
for (const word of words) {
|
||||
if (!word) continue;
|
||||
const piece = pendingIndent + word;
|
||||
pendingIndent = '';
|
||||
const wordRaw = rawTokens(piece);
|
||||
if (wordRaw > atomMax) {
|
||||
flush();
|
||||
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
|
||||
group = group ? `${group} ${piece}` : piece;
|
||||
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
|
||||
function atomize(text: string, atomMax: number): Atom[] {
|
||||
const atoms: Atom[] = [];
|
||||
let prevLine = '';
|
||||
for (const para of text.split(/\n{2,}/)) {
|
||||
let firstInPara = true;
|
||||
for (const rawLine of para.split('\n')) {
|
||||
const line = rawLine.trimEnd();
|
||||
if (!line.trim()) continue;
|
||||
let sep = '';
|
||||
let strength = 0;
|
||||
if (atoms.length > 0) {
|
||||
const afterHeading = HEADING_LINE.test(prevLine);
|
||||
const isHeading = HEADING_LINE.test(line);
|
||||
if (firstInPara) {
|
||||
sep = '\n\n';
|
||||
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
|
||||
} else {
|
||||
sep = '\n';
|
||||
if (isHeading) strength = 4;
|
||||
else if (afterHeading) strength = -0.5;
|
||||
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
|
||||
else strength = 0.5;
|
||||
}
|
||||
}
|
||||
firstInPara = false;
|
||||
prevLine = line;
|
||||
|
||||
if (rawTokens(line) <= Math.min(atomMax, 12)) {
|
||||
// Short line: not worth looking for sentences.
|
||||
pushSentence(line, atomMax, atoms, sep, strength);
|
||||
continue;
|
||||
}
|
||||
let firstSentence = true;
|
||||
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
|
||||
if (!sentence) continue;
|
||||
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
|
||||
firstSentence = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return atoms;
|
||||
}
|
||||
|
||||
function joinAtoms(atoms: Atom[], from: number, to: number): string {
|
||||
let text = '';
|
||||
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
|
||||
return text;
|
||||
}
|
||||
|
||||
/** Trailing whole words of `text` worth at most `budget` tokens. */
|
||||
function tailWords(text: string, budget: number): string {
|
||||
let end = text.length;
|
||||
let cut = text.length;
|
||||
let acc = 0;
|
||||
while (end > 0) {
|
||||
let wordEnd = end;
|
||||
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
|
||||
if (wordEnd === 0) break;
|
||||
const space = text.lastIndexOf(' ', wordEnd - 1);
|
||||
const word = text.slice(space + 1, wordEnd);
|
||||
const cost = rawTokens(word) + WHITESPACE_RAW;
|
||||
if (acc + cost > budget) break;
|
||||
acc += cost;
|
||||
cut = space + 1;
|
||||
end = space < 0 ? 0 : space;
|
||||
}
|
||||
return text.slice(cut).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits one oversized section into balanced pieces of at most `bodyBudget`
|
||||
* tokens, cutting at the strongest boundary near the balanced size, and
|
||||
* prefixing each later piece with up to `overlap` tokens of trailing context.
|
||||
*/
|
||||
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
|
||||
const atomMax = Math.max(1, bodyBudget / 4);
|
||||
const atoms = atomize(text, atomMax);
|
||||
const n = atoms.length;
|
||||
const pieces: string[] = [];
|
||||
if (n === 0) return pieces;
|
||||
|
||||
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
|
||||
const suffix = new Float64Array(n + 1);
|
||||
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
|
||||
|
||||
let start = 0;
|
||||
let prevStart = -1;
|
||||
while (start < n) {
|
||||
const remaining = suffix[start] - atoms[start].sepRaw;
|
||||
let end: number;
|
||||
if (remaining <= bodyBudget) {
|
||||
end = n;
|
||||
} else {
|
||||
const goal = remaining / Math.ceil(remaining / bodyBudget);
|
||||
const minFill = goal * 0.6;
|
||||
let acc = 0;
|
||||
let best = -1;
|
||||
let bestStrength = -Infinity;
|
||||
let bestDistance = Infinity;
|
||||
let i = start;
|
||||
for (; i < n; i++) {
|
||||
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
|
||||
if (i > start && acc + add > bodyBudget) break;
|
||||
acc += add;
|
||||
if (i + 1 < n && acc >= minFill) {
|
||||
const strength = atoms[i + 1].strength;
|
||||
const distance = Math.abs(acc - goal);
|
||||
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
|
||||
best = i;
|
||||
bestStrength = strength;
|
||||
bestDistance = distance;
|
||||
}
|
||||
}
|
||||
}
|
||||
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
|
||||
}
|
||||
|
||||
let prefix = '';
|
||||
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
|
||||
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
|
||||
let from = start;
|
||||
let acc = 0;
|
||||
while (from - 1 > prevStart) {
|
||||
const candidate = atoms[from - 1];
|
||||
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
|
||||
if (acc + cost > overlap) break;
|
||||
acc += cost;
|
||||
from--;
|
||||
if (candidate.strength >= 4) break; // do not reach back past a heading
|
||||
}
|
||||
if (from < start) prefix = joinAtoms(atoms, from, start);
|
||||
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
|
||||
}
|
||||
const body = joinAtoms(atoms, start, end);
|
||||
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
|
||||
prevStart = start;
|
||||
start = end;
|
||||
}
|
||||
return pieces;
|
||||
}
|
||||
|
||||
function positiveInt(value: number | undefined, fallback: number, min: number): number {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
|
||||
return Math.max(min, Math.floor(value));
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
text: string;
|
||||
heading: string;
|
||||
page: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds retrieval chunks from the sections of ONE document.
|
||||
*
|
||||
* - Consecutive small sections (short slides, short pages) are packed together
|
||||
* up to `targetTokens`; the chunk keeps the first section's heading and the
|
||||
* full page range.
|
||||
* - A section that fits comfortably under the ceiling stays whole so chunks
|
||||
* line up with pages/slides.
|
||||
* - Larger sections are split into balanced pieces at paragraph, then
|
||||
* sentence, then word boundaries, with `overlapTokens` of repeated context
|
||||
* between consecutive pieces of the same section only.
|
||||
*/
|
||||
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
|
||||
const target = positiveInt(opts.targetTokens, 450, 8);
|
||||
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
|
||||
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
|
||||
/** Sections up to this size are kept in one piece. */
|
||||
const wholeLimit = Math.floor(target + (max - target) / 2);
|
||||
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
|
||||
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
|
||||
|
||||
const chunks: DocChunk[] = [];
|
||||
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return;
|
||||
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
|
||||
};
|
||||
|
||||
let pending: Pending[] = [];
|
||||
let pendingRaw = 0;
|
||||
const flush = () => {
|
||||
if (pending.length === 0) return;
|
||||
let pageStart: number | null = null;
|
||||
let pageEnd: number | null = null;
|
||||
for (const item of pending) {
|
||||
if (item.page === null) continue;
|
||||
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
|
||||
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
|
||||
}
|
||||
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
|
||||
pending = [];
|
||||
pendingRaw = 0;
|
||||
};
|
||||
|
||||
let lastHeading = '';
|
||||
for (const section of sections ?? []) {
|
||||
if (!section || typeof section.text !== 'string') continue;
|
||||
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
|
||||
if (ownHeading) lastHeading = ownHeading;
|
||||
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
|
||||
if (!text) continue;
|
||||
const heading = lastHeading;
|
||||
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
|
||||
const raw = rawTokens(text);
|
||||
|
||||
if (raw > wholeLimit) {
|
||||
flush();
|
||||
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
|
||||
continue;
|
||||
}
|
||||
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
|
||||
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
|
||||
pending.push({ text, heading, page });
|
||||
}
|
||||
flush();
|
||||
return chunks;
|
||||
}
|
||||
420
server/documents/documents.test.ts
Normal file
420
server/documents/documents.test.ts
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
import { strToU8, zipSync } from 'fflate';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
|
||||
import { detectKind, extractDocument } from './extract.ts';
|
||||
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
|
||||
import { ExtractionError, type ExtractedSection } from './types.ts';
|
||||
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
|
||||
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
|
||||
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
|
||||
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
|
||||
|
||||
async function codeOf(promise: Promise<unknown>): Promise<string> {
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ExtractionError);
|
||||
return (e as ExtractionError).code;
|
||||
}
|
||||
throw new Error('expected rejection');
|
||||
}
|
||||
|
||||
describe('detectKind', () => {
|
||||
const pdf = buildPdf([{ lines: ['hello'] }]);
|
||||
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
|
||||
const pptx = buildPptx([{ file: 1, title: 'T' }]);
|
||||
|
||||
it('detects by magic bytes', () => {
|
||||
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
|
||||
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
|
||||
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
|
||||
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
|
||||
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
|
||||
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
|
||||
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
|
||||
});
|
||||
|
||||
it('trusts magic bytes over a wrong extension', () => {
|
||||
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
|
||||
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
|
||||
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
|
||||
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
|
||||
});
|
||||
|
||||
it('uses the extension for text and markdown', () => {
|
||||
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
|
||||
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
|
||||
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
|
||||
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
|
||||
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
|
||||
});
|
||||
|
||||
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
|
||||
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
|
||||
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
|
||||
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
|
||||
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
|
||||
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: PDF', () => {
|
||||
it('extracts multi-page text with page numbers, count and title', async () => {
|
||||
const bytes = buildPdf(
|
||||
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
|
||||
{ title: 'Cardiology Notes' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
||||
expect(r.kind).toBe('pdf');
|
||||
expect(r.pageCount).toBe(2);
|
||||
expect(r.title).toBe('Cardiology Notes');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
expect(r.warnings).toEqual([]);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
|
||||
expect(r.sections[0].text).toContain('Heart failure overview');
|
||||
expect(r.sections[1].text).toContain('Digoxin');
|
||||
});
|
||||
|
||||
it('warns about a page without text but keeps page numbers', async () => {
|
||||
const long = 'Readable text on this page, long enough to avoid OCR. ';
|
||||
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
||||
expect(r.pageCount).toBe(3);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
|
||||
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
|
||||
});
|
||||
|
||||
it('flags an all-empty scanned PDF as needing OCR', async () => {
|
||||
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
|
||||
expect(r.needsOcr).toBe(true);
|
||||
expect(r.sections).toEqual([]);
|
||||
expect(r.pageCount).toBe(2);
|
||||
expect(r.warnings.join(' ')).toMatch(/scanned/);
|
||||
});
|
||||
|
||||
it('reports progress per page', async () => {
|
||||
const seen: Array<[number, number]> = [];
|
||||
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
|
||||
expect(seen).toEqual([
|
||||
[1, 2],
|
||||
[2, 2],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: DOCX', () => {
|
||||
it('turns headings into sections and keeps lists and tables readable', async () => {
|
||||
const bytes = buildDocx(
|
||||
[
|
||||
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
|
||||
{ type: 'p', text: 'Intro paragraph about drugs.' },
|
||||
{ type: 'bullet', text: 'Beta blockers' },
|
||||
{ type: 'bullet', text: 'ACE inhibitors' },
|
||||
{ type: 'heading', level: 2, text: 'Dosing' },
|
||||
{ type: 'number', text: 'Check heart rate' },
|
||||
{ type: 'number', text: 'Give the dose' },
|
||||
{
|
||||
type: 'table',
|
||||
rows: [
|
||||
['Drug', 'Dose'],
|
||||
['Digoxin', '0.25 mg'],
|
||||
],
|
||||
},
|
||||
],
|
||||
{ title: 'Drug Guide' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.docx' });
|
||||
expect(r.kind).toBe('docx');
|
||||
expect(r.title).toBe('Drug Guide');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
|
||||
const first = r.sections[0].text;
|
||||
expect(first).toContain('Intro paragraph');
|
||||
expect(first).toContain('Beta blockers');
|
||||
expect(first).toContain('ACE inhibitors');
|
||||
const second = r.sections[1].text;
|
||||
expect(second).toContain('Check heart rate');
|
||||
expect(second).toContain('Digoxin');
|
||||
expect(second).toContain('0.25 mg');
|
||||
});
|
||||
|
||||
it('rejects a docx with no text as empty', async () => {
|
||||
const bytes = buildDocx([{ type: 'p', text: '' }]);
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: PPTX', () => {
|
||||
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
|
||||
const bytes = buildPptx(
|
||||
[
|
||||
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
|
||||
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
|
||||
{ file: 2 },
|
||||
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
|
||||
],
|
||||
{ title: 'Deck Title' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pptx' });
|
||||
expect(r.kind).toBe('pptx');
|
||||
expect(r.title).toBe('Deck Title');
|
||||
expect(r.pageCount).toBe(4);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
|
||||
expect(r.sections[0].text).toContain('Welcome');
|
||||
expect(r.sections[0].text).toContain('Sub point');
|
||||
expect(r.sections[0].text).toContain('Say hello slowly');
|
||||
expect(r.sections[1].text).toContain('Free text box content here');
|
||||
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
|
||||
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
|
||||
});
|
||||
|
||||
it('reads tables', async () => {
|
||||
const table = [
|
||||
['A', 'B'],
|
||||
['x', 'y'],
|
||||
];
|
||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
|
||||
expect(allText(r)).toContain('x');
|
||||
expect(allText(r)).toContain('B');
|
||||
});
|
||||
|
||||
it('warns when most slides have no text', async () => {
|
||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
|
||||
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: text, markdown, image', () => {
|
||||
it('splits markdown by heading and ignores headings inside code fences', async () => {
|
||||
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
|
||||
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
|
||||
expect(r.kind).toBe('markdown');
|
||||
expect(r.title).toBe('Title');
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
|
||||
expect(r.sections[1].text).toContain('# not a heading');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
});
|
||||
|
||||
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
|
||||
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
|
||||
expect(r.kind).toBe('text');
|
||||
expect(allText(r)).toContain('line two');
|
||||
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
|
||||
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
|
||||
expect(allText(r2)).toBe('café');
|
||||
expect(r2.warnings.length).toBe(1);
|
||||
});
|
||||
|
||||
it('groups long plain text into several bounded sections', async () => {
|
||||
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
|
||||
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
|
||||
expect(r.sections.length).toBeGreaterThan(1);
|
||||
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
|
||||
});
|
||||
|
||||
it('returns images with needsOcr and no sections', async () => {
|
||||
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
|
||||
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: errors', () => {
|
||||
it('rejects an empty file', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
|
||||
});
|
||||
|
||||
it('rejects whitespace-only text as empty', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
|
||||
});
|
||||
|
||||
it('rejects unsupported types and legacy .doc', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
|
||||
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
|
||||
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
|
||||
});
|
||||
|
||||
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
|
||||
const marker = Buffer.from('EncryptedPackage', 'utf16le');
|
||||
const bytes = Uint8Array.from([...OLE, ...marker]);
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
|
||||
});
|
||||
|
||||
it('rejects corrupt PDFs', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects an encrypted PDF', async () => {
|
||||
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
|
||||
});
|
||||
|
||||
it('rejects a corrupt zip posing as docx/pptx', async () => {
|
||||
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
|
||||
const broken = good.slice(0, Math.floor(good.length / 2));
|
||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
|
||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects a pptx zip that has no readable slides', async () => {
|
||||
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects a zip bomb whose headers lie about entry size', async () => {
|
||||
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
|
||||
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
|
||||
expect(['too_large', 'corrupt']).toContain(code);
|
||||
}, 30000);
|
||||
|
||||
it('honours an already-aborted signal', async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('chunkSections', () => {
|
||||
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
|
||||
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
|
||||
|
||||
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
|
||||
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
|
||||
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
|
||||
page: i + 1,
|
||||
}));
|
||||
const chunks = chunkSections(sections, opts);
|
||||
expect(chunks.length).toBeGreaterThan(12);
|
||||
chunks.forEach((c, i) => {
|
||||
expect(c.seq).toBe(i);
|
||||
expect(c.tokens).toBe(estimateTokens(c.text));
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
});
|
||||
});
|
||||
|
||||
it('packs small sections together with correct page range and first heading', () => {
|
||||
const sections: ExtractedSection[] = [
|
||||
{ text: 'Alpha text.', page: 1, heading: 'First' },
|
||||
{ text: 'Beta text.', page: 2 },
|
||||
{ text: 'Gamma text.', page: 3, heading: 'Third' },
|
||||
];
|
||||
const chunks = chunkSections(sections, opts);
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
|
||||
expect(chunks[0].text).toContain('Gamma');
|
||||
});
|
||||
|
||||
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
|
||||
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
|
||||
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) {
|
||||
expect(c.pageStart).not.toBeNull();
|
||||
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
|
||||
}
|
||||
expect(chunks[0].pageStart).toBe(1);
|
||||
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
|
||||
});
|
||||
|
||||
it('uses null pages and empty heading when sections have none', () => {
|
||||
const [c] = chunkSections([{ text: 'No pages here.' }]);
|
||||
expect(c.pageStart).toBeNull();
|
||||
expect(c.pageEnd).toBeNull();
|
||||
expect(c.heading).toBe('');
|
||||
});
|
||||
|
||||
it('splits oversized sections without cutting words', () => {
|
||||
const text = words(800, 'term');
|
||||
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
const vocabulary = new Set(text.split(' '));
|
||||
for (const c of chunks) {
|
||||
expect(c.pageStart).toBe(5);
|
||||
expect(c.pageEnd).toBe(5);
|
||||
expect(c.heading).toBe('Big');
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
|
||||
}
|
||||
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
|
||||
expect(seen.size).toBe(800);
|
||||
});
|
||||
|
||||
it('survives a giant unbroken string within the limit', () => {
|
||||
const blob = 'x'.repeat(200_000);
|
||||
const t0 = Date.now();
|
||||
const chunks = chunkSections([{ text: blob }], opts);
|
||||
expect(Date.now() - t0).toBeLessThan(3000);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
|
||||
});
|
||||
|
||||
it('does not split surrogate pairs in unbroken emoji runs', () => {
|
||||
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) {
|
||||
expect(wellFormed(c.text)).toBe(true);
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
}
|
||||
});
|
||||
|
||||
it('is deterministic', () => {
|
||||
const sections = [
|
||||
{ text: words(900), page: 1, heading: 'H' },
|
||||
{ text: 'Short one.', page: 2 },
|
||||
];
|
||||
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
|
||||
});
|
||||
|
||||
it('drops empty and whitespace-only sections', () => {
|
||||
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
|
||||
const chunks = chunkSections([
|
||||
{ text: ' ', page: 1 },
|
||||
{ text: 'Real content.', page: 2 },
|
||||
{ text: '\n', page: 3 },
|
||||
]);
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
|
||||
});
|
||||
|
||||
it('chunks about 2 MB of text quickly', () => {
|
||||
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
|
||||
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
|
||||
const t0 = Date.now();
|
||||
const chunks = chunkSections([{ text }]);
|
||||
expect(Date.now() - t0).toBeLessThan(3000);
|
||||
expect(chunks.length).toBeGreaterThan(100);
|
||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanText', () => {
|
||||
it('joins words hyphenated across a line break', () => {
|
||||
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
|
||||
});
|
||||
|
||||
it('keeps real compounds across a line break', () => {
|
||||
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
|
||||
});
|
||||
|
||||
it('collapses 3+ newlines to a paragraph break', () => {
|
||||
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
||||
expect(cleanText('a\n\nb')).toBe('a\n\nb');
|
||||
});
|
||||
|
||||
it('strips control characters but keeps tabs and newlines', () => {
|
||||
expect(cleanText('he\u0000l\u0007lo world\ttab\nnext')).toBe('hello world\ttab\nnext');
|
||||
});
|
||||
|
||||
it('normalises CRLF and trims', () => {
|
||||
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
|
||||
expect(cleanText('')).toBe('');
|
||||
});
|
||||
|
||||
it('is idempotent', () => {
|
||||
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny z');
|
||||
expect(cleanText(once)).toBe(once);
|
||||
});
|
||||
});
|
||||
105
server/documents/docx.ts
Normal file
105
server/documents/docx.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
|
||||
import { zipSync } from 'fflate';
|
||||
import mammoth from 'mammoth';
|
||||
import { cleanText } from './chunk.ts';
|
||||
import { htmlToBlocks } from './markup.ts';
|
||||
import { readCoreTitle, type OfficeContext } from './pptx.ts';
|
||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
||||
import { ZipReader } from './zip.ts';
|
||||
|
||||
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
|
||||
const MAX_WARNINGS = 3;
|
||||
|
||||
/**
|
||||
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
|
||||
* into a fresh stored zip. Every byte went through the size-limited reader, so
|
||||
* mammoth's own unzip can never be handed a bomb, and it never spends time
|
||||
* base64-encoding images we would throw away.
|
||||
*/
|
||||
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
|
||||
const files: Record<string, Uint8Array> = {};
|
||||
let images = 0;
|
||||
for (const entry of zip.entries) {
|
||||
const name = entry.name;
|
||||
if (name.endsWith('/')) continue;
|
||||
if (/^word\/media\//i.test(name)) {
|
||||
images++;
|
||||
continue;
|
||||
}
|
||||
if (!/\.(xml|rels)$/i.test(name)) continue;
|
||||
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
|
||||
const data = zip.read(name);
|
||||
if (data) files[name] = data;
|
||||
}
|
||||
return { data: zipSync(files, { level: 0 }), images };
|
||||
}
|
||||
|
||||
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
|
||||
const seen = new Set<string>();
|
||||
for (const item of messages) {
|
||||
const message = String(item?.message ?? '').trim();
|
||||
// Unknown style names never lose text; they are noise for a reader.
|
||||
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
|
||||
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
|
||||
}
|
||||
const unique = [...seen];
|
||||
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
|
||||
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
||||
const zip = new ZipReader(bytes, ctx.limits);
|
||||
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
|
||||
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
|
||||
}
|
||||
const { data, images } = sanitisedPackage(zip);
|
||||
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
||||
|
||||
const converted = await mammoth.convertToHtml(
|
||||
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
|
||||
{
|
||||
styleMap: STYLE_MAP,
|
||||
ignoreEmptyParagraphs: true,
|
||||
// Images are not read locally; never decode them.
|
||||
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
|
||||
},
|
||||
);
|
||||
|
||||
const sections: ExtractedSection[] = [];
|
||||
let heading: string | undefined;
|
||||
let parts: string[] = [];
|
||||
const flush = () => {
|
||||
const text = cleanText(parts.join('\n\n'));
|
||||
parts = [];
|
||||
if (!text) return;
|
||||
const section: ExtractedSection = { text };
|
||||
if (heading) section.heading = heading;
|
||||
sections.push(section);
|
||||
};
|
||||
for (const block of htmlToBlocks(converted.value)) {
|
||||
if (block.level >= 1 && block.level <= 3) {
|
||||
flush();
|
||||
heading = block.text.slice(0, 200);
|
||||
}
|
||||
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
|
||||
}
|
||||
flush();
|
||||
ctx.onProgress?.(1, 1);
|
||||
|
||||
const warnings = friendlyWarnings(converted.messages ?? []);
|
||||
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
|
||||
let needsOcr = false;
|
||||
if (totalChars < 25) {
|
||||
if (images === 0) {
|
||||
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
|
||||
}
|
||||
needsOcr = true;
|
||||
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
|
||||
}
|
||||
|
||||
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
|
||||
const title = readCoreTitle(zip);
|
||||
if (title) result.title = title;
|
||||
return result;
|
||||
}
|
||||
362
server/documents/extract.ts
Normal file
362
server/documents/extract.ts
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
/**
|
||||
* Document text extraction: kind detection plus one extractor per format.
|
||||
* Every failure surfaces as an ExtractionError with a user-facing message.
|
||||
*/
|
||||
import { cleanText } from './chunk.ts';
|
||||
import { extractDocx } from './docx.ts';
|
||||
import { extractPdf } from './pdf.ts';
|
||||
import { extractPptx } from './pptx.ts';
|
||||
import { ExtractionError, type DocKind, type ExtractedSection, type ExtractResult } from './types.ts';
|
||||
import { listZip, ZipError, type ZipLimits } from './zip.ts';
|
||||
|
||||
/** Largest upload accepted (100 MB). */
|
||||
export const MAX_DOCUMENT_BYTES = 100 * 1024 * 1024;
|
||||
|
||||
/** Zip-bomb guards for DOCX/PPTX: uncompressed bytes per entry and in total. */
|
||||
const ZIP_LIMITS: ZipLimits = { maxEntryBytes: 64 * 1024 * 1024, maxTotalBytes: 300 * 1024 * 1024 };
|
||||
|
||||
export interface ExtractInput {
|
||||
bytes: Uint8Array;
|
||||
filename: string;
|
||||
mime?: string;
|
||||
/** Called as pages/slides are processed (PDF and PPTX report every page; other formats once). */
|
||||
onProgress?: (done: number, total: number) => void;
|
||||
/** Aborting rejects with ExtractionError('failed') at the next page boundary. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
const TEXT_EXTENSIONS = new Set(['txt', 'text', 'csv', 'tsv']);
|
||||
const MARKDOWN_EXTENSIONS = new Set(['md', 'markdown']);
|
||||
const TEXT_MIMES = new Set(['text/plain', 'text/csv', 'text/tab-separated-values']);
|
||||
const MARKDOWN_MIMES = new Set(['text/markdown', 'text/x-markdown']);
|
||||
const DOCX_MIME = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document';
|
||||
const PPTX_MIME = 'application/vnd.openxmlformats-officedocument.presentationml.presentation';
|
||||
const SNIFF_BYTES = 256 * 1024;
|
||||
|
||||
function extensionOf(filename: string): string {
|
||||
const base = (filename ?? '').split(/[\\/]/).pop() ?? '';
|
||||
const dot = base.lastIndexOf('.');
|
||||
return dot > 0 ? base.slice(dot + 1).toLowerCase() : '';
|
||||
}
|
||||
|
||||
function baseMime(mime: string | undefined): string {
|
||||
return (mime ?? '').split(';')[0].trim().toLowerCase();
|
||||
}
|
||||
|
||||
function startsWith(bytes: Uint8Array, signature: number[], offset = 0): boolean {
|
||||
if (bytes.length < offset + signature.length) return false;
|
||||
for (let i = 0; i < signature.length; i++) if (bytes[offset + i] !== signature[i]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function isImage(bytes: Uint8Array): boolean {
|
||||
return (
|
||||
startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) || // PNG
|
||||
startsWith(bytes, [0xff, 0xd8, 0xff]) || // JPEG
|
||||
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x37, 0x61]) || // GIF87a
|
||||
startsWith(bytes, [0x47, 0x49, 0x46, 0x38, 0x39, 0x61]) || // GIF89a
|
||||
(startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes, [0x57, 0x45, 0x42, 0x50], 8)) // RIFF....WEBP
|
||||
);
|
||||
}
|
||||
|
||||
/** `%PDF-` may be preceded by a little junk; readers accept it within the first 1024 bytes. */
|
||||
function hasPdfHeader(bytes: Uint8Array): boolean {
|
||||
const limit = Math.min(bytes.length - 5, 1024);
|
||||
for (let i = 0; i <= limit; i++) {
|
||||
if (bytes[i] === 0x25 && bytes[i + 1] === 0x50 && bytes[i + 2] === 0x44 && bytes[i + 3] === 0x46 && bytes[i + 4] === 0x2d) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isZip(bytes: Uint8Array): boolean {
|
||||
return startsWith(bytes, [0x50, 0x4b, 0x03, 0x04]) || startsWith(bytes, [0x50, 0x4b, 0x05, 0x06]) || startsWith(bytes, [0x50, 0x4b, 0x07, 0x08]);
|
||||
}
|
||||
|
||||
/** OLE compound file: legacy .doc/.ppt/.xls, and also the wrapper of password-protected OOXML. */
|
||||
function isOle(bytes: Uint8Array): boolean {
|
||||
return startsWith(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]);
|
||||
}
|
||||
|
||||
function isUtf16Bom(bytes: Uint8Array): boolean {
|
||||
return bytes.length >= 2 && ((bytes[0] === 0xff && bytes[1] === 0xfe) || (bytes[0] === 0xfe && bytes[1] === 0xff));
|
||||
}
|
||||
|
||||
function isValidUtf8(bytes: Uint8Array, allowTruncatedTail: boolean): boolean {
|
||||
try {
|
||||
new TextDecoder('utf-8', { fatal: true }).decode(bytes, { stream: allowTruncatedTail });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** No NULs or other control bytes, and not dominated by high bytes: plausibly legacy 8-bit text. */
|
||||
function looksLikeSingleByteText(sample: Uint8Array): boolean {
|
||||
let high = 0;
|
||||
for (let i = 0; i < sample.length; i++) {
|
||||
const b = sample[i];
|
||||
if (b < 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d && b !== 0x0c) return false;
|
||||
if (b === 0x7f) return false;
|
||||
if (b >= 0x80) high++;
|
||||
}
|
||||
return high <= sample.length * 0.3;
|
||||
}
|
||||
|
||||
function isProbablyText(bytes: Uint8Array): boolean {
|
||||
if (isUtf16Bom(bytes)) return bytes.length % 2 === 0;
|
||||
const sample = bytes.length > SNIFF_BYTES ? bytes.subarray(0, SNIFF_BYTES) : bytes;
|
||||
if (sample.includes(0)) return false;
|
||||
return isValidUtf8(sample, sample.length < bytes.length) || looksLikeSingleByteText(sample);
|
||||
}
|
||||
|
||||
/**
|
||||
* Works out what a file is. Magic bytes win over the name; the extension and
|
||||
* MIME type are only a fallback. Returns null for anything unsupported
|
||||
* (legacy .doc/.ppt, spreadsheets, archives, executables, binary junk).
|
||||
*/
|
||||
export function detectKind(filename: string, mime: string | undefined, bytes: Uint8Array): DocKind | null {
|
||||
const ext = extensionOf(filename);
|
||||
const type = baseMime(mime);
|
||||
if (!bytes || bytes.length === 0) return null;
|
||||
|
||||
if (hasPdfHeader(bytes) && (startsWith(bytes, [0x25, 0x50, 0x44, 0x46]) || ext === 'pdf' || type === 'application/pdf')) return 'pdf';
|
||||
if (isImage(bytes)) return 'image';
|
||||
|
||||
if (isZip(bytes)) {
|
||||
let names: string[] | null = null;
|
||||
try {
|
||||
names = listZip(bytes).map((entry) => entry.name);
|
||||
} catch {
|
||||
names = null;
|
||||
}
|
||||
if (names) {
|
||||
if (names.some((name) => /^word\/document\d*\.xml$/i.test(name))) return 'docx';
|
||||
if (names.includes('ppt/presentation.xml')) return 'pptx';
|
||||
return null; // xlsx, odt, epub, jar, plain zip...
|
||||
}
|
||||
// Unreadable archive: trust the name so extraction can report it as damaged.
|
||||
if (ext === 'docx' || type === DOCX_MIME) return 'docx';
|
||||
if (ext === 'pptx' || type === PPTX_MIME) return 'pptx';
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isOle(bytes)) {
|
||||
// Password-protected .docx/.pptx are OLE containers; extraction explains that.
|
||||
if (ext === 'docx') return 'docx';
|
||||
if (ext === 'pptx') return 'pptx';
|
||||
return null;
|
||||
}
|
||||
|
||||
// No recognisable signature: fall back to the name / declared type.
|
||||
if (ext === 'pdf' || (!ext && type === 'application/pdf')) return 'pdf';
|
||||
if (ext === 'docx' || (!ext && type === DOCX_MIME)) return 'docx';
|
||||
if (ext === 'pptx' || (!ext && type === PPTX_MIME)) return 'pptx';
|
||||
if (MARKDOWN_EXTENSIONS.has(ext) || (!ext && MARKDOWN_MIMES.has(type))) return isProbablyText(bytes) ? 'markdown' : null;
|
||||
if (TEXT_EXTENSIONS.has(ext) || (!ext && TEXT_MIMES.has(type))) return isProbablyText(bytes) ? 'text' : null;
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Text and Markdown
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function decodeText(bytes: Uint8Array, warnings: string[]): string {
|
||||
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
||||
return new TextDecoder('utf-8').decode(bytes.subarray(3));
|
||||
}
|
||||
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) return new TextDecoder('utf-16le').decode(bytes.subarray(2));
|
||||
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) return new TextDecoder('utf-16be').decode(bytes.subarray(2));
|
||||
try {
|
||||
return new TextDecoder('utf-8', { fatal: true }).decode(bytes);
|
||||
} catch {
|
||||
if (bytes.includes(0)) {
|
||||
throw new ExtractionError('unsupported', "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.");
|
||||
}
|
||||
warnings.push('This file is not saved as UTF-8, so some special characters may look wrong.');
|
||||
return new TextDecoder('windows-1252').decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/** Target size of a plain-text section, in characters. */
|
||||
const TEXT_SECTION_CHARS = 6000;
|
||||
|
||||
function splitPlainText(text: string): ExtractedSection[] {
|
||||
const sections: ExtractedSection[] = [];
|
||||
let current: string[] = [];
|
||||
let size = 0;
|
||||
const flush = (separator: string) => {
|
||||
if (current.length === 0) return;
|
||||
sections.push({ text: current.join(separator) });
|
||||
current = [];
|
||||
size = 0;
|
||||
};
|
||||
for (const block of text.split(/\n{2,}/)) {
|
||||
if (!block.trim()) continue;
|
||||
if (block.length > TEXT_SECTION_CHARS) {
|
||||
// One huge block (CSV, logs, text without blank lines): fall back to line boundaries.
|
||||
flush('\n\n');
|
||||
for (const line of block.split('\n')) {
|
||||
if (size > 0 && size + line.length + 1 > TEXT_SECTION_CHARS) flush('\n');
|
||||
current.push(line);
|
||||
size += line.length + 1;
|
||||
}
|
||||
flush('\n');
|
||||
continue;
|
||||
}
|
||||
if (size > 0 && size + block.length + 2 > TEXT_SECTION_CHARS) flush('\n\n');
|
||||
current.push(block);
|
||||
size += block.length + 2;
|
||||
}
|
||||
flush('\n\n');
|
||||
return sections;
|
||||
}
|
||||
|
||||
function markdownHeadingText(raw: string): string {
|
||||
return raw
|
||||
.replace(/[ \t]+#+[ \t]*$/, '')
|
||||
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\*\*|__|[*`]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 200);
|
||||
}
|
||||
|
||||
function splitMarkdown(text: string): { sections: ExtractedSection[]; title?: string } {
|
||||
const sections: ExtractedSection[] = [];
|
||||
let title: string | undefined;
|
||||
let heading: string | undefined;
|
||||
let lines: string[] = [];
|
||||
let fence: string | null = null;
|
||||
const flush = () => {
|
||||
const body = lines.join('\n').trim();
|
||||
lines = [];
|
||||
if (!body) return;
|
||||
const section: ExtractedSection = { text: body };
|
||||
if (heading) section.heading = heading;
|
||||
sections.push(section);
|
||||
};
|
||||
for (const line of text.split('\n')) {
|
||||
const fenceMatch = line.length < 400 ? /^ {0,3}(`{3,}|~{3,})/.exec(line) : null;
|
||||
if (fenceMatch) {
|
||||
const marker = fenceMatch[1][0];
|
||||
if (fence === null) fence = marker;
|
||||
else if (fence === marker) fence = null;
|
||||
} else if (fence === null && line.length < 400) {
|
||||
const match = /^ {0,3}(#{1,3})[ \t]+(\S.*)$/.exec(line);
|
||||
if (match) {
|
||||
const headingText = markdownHeadingText(match[2]);
|
||||
if (headingText) {
|
||||
flush();
|
||||
heading = headingText;
|
||||
if (match[1].length === 1 && title === undefined && sections.length === 0) title = headingText;
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
flush();
|
||||
return { sections, title };
|
||||
}
|
||||
|
||||
function extractText(bytes: Uint8Array, kind: 'text' | 'markdown'): ExtractResult {
|
||||
const warnings: string[] = [];
|
||||
const text = cleanText(decodeText(bytes, warnings));
|
||||
if (!text) throw new ExtractionError('empty', "This file doesn't contain any text.");
|
||||
if (kind === 'markdown') {
|
||||
const { sections, title } = splitMarkdown(text);
|
||||
const result: ExtractResult = { kind, sections, warnings, needsOcr: false };
|
||||
if (title) result.title = title;
|
||||
return result;
|
||||
}
|
||||
return { kind, sections: splitPlainText(text), warnings, needsOcr: false };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Entry point
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const UTF16_ENCRYPTED_PACKAGE = Buffer.from('EncryptedPackage', 'utf16le');
|
||||
|
||||
function unsupportedMessage(filename: string, bytes: Uint8Array): string {
|
||||
const ext = extensionOf(filename);
|
||||
if (ext === 'doc' || ext === 'dot') return "Old Word .doc files aren't supported. Open it in Word, save it as .docx or PDF, and upload that instead.";
|
||||
if (ext === 'ppt' || ext === 'pps') return "Old PowerPoint .ppt files aren't supported. Open it in PowerPoint, save it as .pptx or PDF, and upload that instead.";
|
||||
if (ext === 'xls' || ext === 'xlsx') return "Spreadsheets aren't supported. Export the sheet as PDF or CSV and upload that instead.";
|
||||
if (ext === 'pages' || ext === 'key' || ext === 'odt' || ext === 'odp' || ext === 'rtf' || ext === 'epub') {
|
||||
return `.${ext} files aren't supported. Export it as PDF or Word (.docx) and upload that instead.`;
|
||||
}
|
||||
if (TEXT_EXTENSIONS.has(ext) || MARKDOWN_EXTENSIONS.has(ext)) {
|
||||
return "This file doesn't look like plain text. Upload a PDF, Word, PowerPoint, text or Markdown file.";
|
||||
}
|
||||
if (isOle(bytes)) return "Old Office files (.doc, .ppt) aren't supported. Save it as .docx, .pptx or PDF and upload that instead.";
|
||||
return "This file type isn't supported. Upload a PDF, Word (.docx), PowerPoint (.pptx), text, Markdown or image file.";
|
||||
}
|
||||
|
||||
function fromZipError(error: ZipError, kind: DocKind): ExtractionError {
|
||||
const label = kind === 'pptx' ? 'PowerPoint file' : 'Word document';
|
||||
if (error.code === 'too_large') {
|
||||
return new ExtractionError('too_large', `This ${label} is too large to read once unpacked. Try splitting it into smaller files or exporting it as PDF.`, { cause: error });
|
||||
}
|
||||
return new ExtractionError('corrupt', `This ${label} looks damaged and could not be opened. Try saving it again and re-uploading.`, { cause: error });
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the text of a document as ordered sections. Images return no
|
||||
* sections and `needsOcr: true`; the caller runs vision transcription.
|
||||
*/
|
||||
export async function extractDocument(input: ExtractInput): Promise<ExtractResult> {
|
||||
const bytes = input?.bytes;
|
||||
const filename = typeof input?.filename === 'string' ? input.filename : '';
|
||||
if (!(bytes instanceof Uint8Array) || bytes.length === 0) {
|
||||
throw new ExtractionError('empty', 'This file is empty. Check the file and upload it again.');
|
||||
}
|
||||
if (bytes.length > MAX_DOCUMENT_BYTES) {
|
||||
throw new ExtractionError('too_large', `This file is too large. The limit is ${Math.round(MAX_DOCUMENT_BYTES / (1024 * 1024))} MB.`);
|
||||
}
|
||||
if (input.signal?.aborted) {
|
||||
throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: input.signal.reason });
|
||||
}
|
||||
|
||||
let kind: DocKind | null;
|
||||
try {
|
||||
kind = detectKind(filename, input.mime, bytes);
|
||||
} catch (error) {
|
||||
throw new ExtractionError('failed', "We couldn't work out what kind of file this is.", { cause: error });
|
||||
}
|
||||
if (kind === null) throw new ExtractionError('unsupported', unsupportedMessage(filename, bytes));
|
||||
|
||||
try {
|
||||
switch (kind) {
|
||||
case 'pdf':
|
||||
return await extractPdf(bytes, { onProgress: input.onProgress, signal: input.signal });
|
||||
case 'docx':
|
||||
case 'pptx': {
|
||||
if (isOle(bytes)) {
|
||||
const view = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (view.indexOf(UTF16_ENCRYPTED_PACKAGE) >= 0) {
|
||||
throw new ExtractionError('encrypted', `This ${kind === 'pptx' ? 'PowerPoint file' : 'Word document'} is password-protected. Remove the password and upload it again.`);
|
||||
}
|
||||
throw new ExtractionError('unsupported', unsupportedMessage(kind === 'pptx' ? 'x.ppt' : 'x.doc', bytes));
|
||||
}
|
||||
const ctx = { limits: ZIP_LIMITS, onProgress: input.onProgress, signal: input.signal };
|
||||
return kind === 'docx' ? await extractDocx(bytes, ctx) : await extractPptx(bytes, ctx);
|
||||
}
|
||||
case 'text':
|
||||
case 'markdown': {
|
||||
const result = extractText(bytes, kind);
|
||||
input.onProgress?.(1, 1);
|
||||
return result;
|
||||
}
|
||||
case 'image':
|
||||
return { kind: 'image', sections: [], needsOcr: true, warnings: [] };
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof ExtractionError) throw error;
|
||||
if (error instanceof ZipError) throw fromZipError(error, kind);
|
||||
const label = kind === 'docx' ? 'Word document' : kind === 'pptx' ? 'PowerPoint file' : kind === 'pdf' ? 'PDF' : 'file';
|
||||
if (kind === 'docx' || kind === 'pptx') {
|
||||
throw new ExtractionError('corrupt', `This ${label} looks damaged and could not be read. Try saving it again and re-uploading.`, { cause: error });
|
||||
}
|
||||
throw new ExtractionError('failed', `We couldn't read this ${label}. Try uploading it again.`, { cause: error });
|
||||
}
|
||||
}
|
||||
272
server/documents/markup.ts
Normal file
272
server/documents/markup.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
/** Small markup helpers shared by the DOCX and PPTX extractors. */
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
};
|
||||
|
||||
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
|
||||
export function decodeEntities(text: string): string {
|
||||
if (!text.includes('&')) return text;
|
||||
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
|
||||
return String.fromCodePoint(code);
|
||||
}
|
||||
return NAMED_ENTITIES[body] ?? match;
|
||||
});
|
||||
}
|
||||
|
||||
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
|
||||
export function renderTable(rows: string[][]): string {
|
||||
const cleaned = rows
|
||||
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
|
||||
.filter((row) => row.some((cell) => cell.length > 0));
|
||||
if (cleaned.length === 0) return '';
|
||||
const width = Math.max(...cleaned.map((row) => row.length));
|
||||
const line = (row: string[]) => {
|
||||
const cells = row.slice();
|
||||
while (cells.length < width) cells.push('');
|
||||
return `| ${cells.join(' | ')} |`;
|
||||
};
|
||||
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
|
||||
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTML (as produced by mammoth) -> Markdown blocks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface MarkdownBlock {
|
||||
/** 1-6 for headings, 0 for everything else. */
|
||||
level: number;
|
||||
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
interface HtmlElement {
|
||||
tag: string;
|
||||
children: HtmlNode[];
|
||||
}
|
||||
type HtmlNode = HtmlElement | string;
|
||||
|
||||
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
|
||||
const HEADING_TAG = /^h[1-6]$/;
|
||||
|
||||
function parseHtml(html: string): HtmlElement {
|
||||
const root: HtmlElement = { tag: '#root', children: [] };
|
||||
const stack: HtmlElement[] = [root];
|
||||
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = tokens.exec(html)) !== null) {
|
||||
const top = stack[stack.length - 1];
|
||||
if (match[4] !== undefined) {
|
||||
top.children.push(decodeEntities(match[4]));
|
||||
} else if (match[2] === undefined) {
|
||||
top.children.push('<');
|
||||
} else {
|
||||
const tag = match[2].toLowerCase();
|
||||
if (match[1]) {
|
||||
// Close the nearest matching open element; ignore stray closers.
|
||||
for (let i = stack.length - 1; i > 0; i--) {
|
||||
if (stack[i].tag === tag) {
|
||||
stack.length = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const element: HtmlElement = { tag, children: [] };
|
||||
top.children.push(element);
|
||||
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function collapse(text: string): string {
|
||||
return text.replace(/[ \t\r\n]+/g, ' ');
|
||||
}
|
||||
|
||||
function wrap(marker: string, inner: string): string {
|
||||
const core = inner.trim();
|
||||
if (!core) return inner;
|
||||
const lead = inner.slice(0, inner.length - inner.trimStart().length);
|
||||
const trail = inner.slice(inner.trimEnd().length);
|
||||
return `${lead}${marker}${core}${marker}${trail}`;
|
||||
}
|
||||
|
||||
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
|
||||
function inlineText(nodes: HtmlNode[], plain: boolean): string {
|
||||
let out = '';
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') {
|
||||
out += collapse(node);
|
||||
continue;
|
||||
}
|
||||
switch (node.tag) {
|
||||
case 'br':
|
||||
out += '\n';
|
||||
break;
|
||||
case 'img':
|
||||
case 'hr':
|
||||
break;
|
||||
case 'strong':
|
||||
case 'b':
|
||||
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
|
||||
break;
|
||||
case 'em':
|
||||
case 'i':
|
||||
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
|
||||
break;
|
||||
case 'a': {
|
||||
const text = inlineText(node.children, plain);
|
||||
// mammoth's footnote back-references
|
||||
if (text.trim() !== '↑') out += text;
|
||||
break;
|
||||
}
|
||||
case 'p':
|
||||
case 'li':
|
||||
case 'tr':
|
||||
case 'td':
|
||||
case 'th':
|
||||
case 'ul':
|
||||
case 'ol':
|
||||
case 'table': {
|
||||
const inner = inlineText(node.children, plain).trim();
|
||||
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
|
||||
break;
|
||||
}
|
||||
default:
|
||||
out += inlineText(node.children, plain);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function tidyInline(text: string): string {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/ {2,}/g, ' ').trim())
|
||||
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
|
||||
const ordered = list.tag === 'ol';
|
||||
let counter = 0;
|
||||
for (const item of list.children) {
|
||||
if (typeof item === 'string' || item.tag !== 'li') continue;
|
||||
counter++;
|
||||
const marker = ordered ? `${counter}. ` : '- ';
|
||||
const inlineNodes: HtmlNode[] = [];
|
||||
const nested: HtmlElement[] = [];
|
||||
const paragraphs: string[] = [];
|
||||
const flushInline = () => {
|
||||
const text = tidyInline(inlineText(inlineNodes, false));
|
||||
if (text) paragraphs.push(text);
|
||||
inlineNodes.length = 0;
|
||||
};
|
||||
for (const child of item.children) {
|
||||
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
|
||||
flushInline();
|
||||
nested.push(child);
|
||||
} else if (typeof child !== 'string' && child.tag === 'p') {
|
||||
flushInline();
|
||||
inlineNodes.push(...child.children);
|
||||
flushInline();
|
||||
} else if (typeof child !== 'string' && child.tag === 'table') {
|
||||
flushInline();
|
||||
const table = renderHtmlTable(child);
|
||||
if (table) paragraphs.push(table);
|
||||
} else {
|
||||
inlineNodes.push(child);
|
||||
}
|
||||
}
|
||||
flushInline();
|
||||
const pad = indent + ' '.repeat(marker.length);
|
||||
const body = paragraphs.join('\n').split('\n');
|
||||
if (body.length > 0 && body[0] !== '') {
|
||||
lines.push(indent + marker + body[0]);
|
||||
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
|
||||
} else if (nested.length === 0) {
|
||||
continue;
|
||||
}
|
||||
for (const child of nested) renderList(child, pad, lines);
|
||||
}
|
||||
}
|
||||
|
||||
function renderHtmlTable(table: HtmlElement): string {
|
||||
const rows: string[][] = [];
|
||||
const visit = (nodes: HtmlNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') continue;
|
||||
if (node.tag === 'tr') {
|
||||
const cells: string[] = [];
|
||||
for (const cell of node.children) {
|
||||
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
|
||||
cells.push(tidyInline(inlineText(cell.children, false)));
|
||||
}
|
||||
rows.push(cells);
|
||||
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
|
||||
visit(node.children);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(table.children);
|
||||
return renderTable(rows);
|
||||
}
|
||||
|
||||
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
|
||||
export function htmlToBlocks(html: string): MarkdownBlock[] {
|
||||
const root = parseHtml(html);
|
||||
const blocks: MarkdownBlock[] = [];
|
||||
const loose: HtmlNode[] = [];
|
||||
const flushLoose = () => {
|
||||
if (loose.length === 0) return;
|
||||
const text = tidyInline(inlineText(loose, false));
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
loose.length = 0;
|
||||
};
|
||||
const visit = (nodes: HtmlNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') {
|
||||
loose.push(node);
|
||||
continue;
|
||||
}
|
||||
if (HEADING_TAG.test(node.tag)) {
|
||||
flushLoose();
|
||||
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
|
||||
if (text) blocks.push({ level: Number(node.tag[1]), text });
|
||||
} else if (node.tag === 'p') {
|
||||
flushLoose();
|
||||
const text = tidyInline(inlineText(node.children, false));
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
} else if (node.tag === 'ul' || node.tag === 'ol') {
|
||||
flushLoose();
|
||||
const lines: string[] = [];
|
||||
renderList(node, '', lines);
|
||||
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
|
||||
} else if (node.tag === 'table') {
|
||||
flushLoose();
|
||||
const text = renderHtmlTable(node);
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
|
||||
flushLoose();
|
||||
visit(node.children);
|
||||
} else {
|
||||
loose.push(node);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(root.children);
|
||||
flushLoose();
|
||||
return blocks;
|
||||
}
|
||||
249
server/documents/pdf.ts
Normal file
249
server/documents/pdf.ts
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
/** PDF extraction through unpdf (pdf.js, serverless build): one section per page. */
|
||||
import { getDocumentProxy } from 'unpdf';
|
||||
import { cleanText, stripRepeatedPageLines } from './chunk.ts';
|
||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
||||
|
||||
export interface PdfContext {
|
||||
onProgress?: (done: number, total: number) => void;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** Below this average of characters per page the PDF is treated as a scan. */
|
||||
const OCR_CHARS_PER_PAGE = 25;
|
||||
/** Release pdf.js caches (fonts, parsed objects) every this many pages. */
|
||||
const CLEANUP_EVERY = 40;
|
||||
|
||||
interface TextItemLike {
|
||||
str?: unknown;
|
||||
hasEOL?: unknown;
|
||||
transform?: unknown;
|
||||
height?: unknown;
|
||||
}
|
||||
|
||||
interface Line {
|
||||
text: string;
|
||||
y: number | null;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds page text from pdf.js text items. Items arrive in content-stream
|
||||
* order (which keeps multi-column pages readable) with `hasEOL` marking line
|
||||
* ends; the vertical gaps between lines are then used to restore paragraph
|
||||
* breaks, which pdf.js does not report.
|
||||
*/
|
||||
export function textItemsToText(items: readonly TextItemLike[]): string {
|
||||
const lines: Line[] = [];
|
||||
let text = '';
|
||||
let y: number | null = null;
|
||||
let height = 0;
|
||||
const endLine = () => {
|
||||
lines.push({ text, y, height });
|
||||
text = '';
|
||||
y = null;
|
||||
height = 0;
|
||||
};
|
||||
for (const item of items) {
|
||||
if (!item || typeof item.str !== 'string') continue;
|
||||
const str = item.str;
|
||||
if (str.length > 0) {
|
||||
if (y === null && str.trim().length > 0 && Array.isArray(item.transform)) {
|
||||
const t = item.transform as number[];
|
||||
const upright = Math.abs(t[1] ?? 0) < 1e-3 && Math.abs(t[2] ?? 0) < 1e-3;
|
||||
if (upright && Number.isFinite(t[5])) {
|
||||
y = t[5];
|
||||
const h = typeof item.height === 'number' && item.height > 0 ? item.height : Math.hypot(t[2] ?? 0, t[3] ?? 0);
|
||||
height = Number.isFinite(h) ? h : 0;
|
||||
}
|
||||
}
|
||||
text += str;
|
||||
}
|
||||
if (item.hasEOL === true) endLine();
|
||||
}
|
||||
if (text.length > 0) endLine();
|
||||
|
||||
// Typical line advance on this page (median of the downward steps).
|
||||
const advances: number[] = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const a = lines[i - 1];
|
||||
const b = lines[i];
|
||||
if (a.y === null || b.y === null) continue;
|
||||
const dy = a.y - b.y;
|
||||
const ref = Math.max(a.height, b.height, 1);
|
||||
if (dy > 0 && dy < ref * 4) advances.push(dy);
|
||||
}
|
||||
advances.sort((p, q) => p - q);
|
||||
const median = advances.length >= 3 ? advances[Math.floor(advances.length / 2)] : 0;
|
||||
|
||||
let out = '';
|
||||
let prev: Line | null = null;
|
||||
for (const line of lines) {
|
||||
const content = line.text.replace(/[ \t\u00A0]+/g, ' ').trim();
|
||||
if (!content) {
|
||||
// An empty line is an explicit gap.
|
||||
if (out && !out.endsWith('\n\n')) out += out.endsWith('\n') ? '\n' : '\n\n';
|
||||
continue;
|
||||
}
|
||||
if (out && !out.endsWith('\n')) {
|
||||
let gap = false;
|
||||
if (prev && prev.y !== null && line.y !== null) {
|
||||
const dy = prev.y - line.y;
|
||||
const ref = Math.max(prev.height, line.height, 1);
|
||||
if (dy < -ref * 2) gap = true; // jumped back up: new column or block
|
||||
else if (median > 0) gap = dy > median * 1.45 && dy > ref * 1.3;
|
||||
else gap = dy > ref * 1.9;
|
||||
}
|
||||
out += gap ? '\n\n' : '\n';
|
||||
}
|
||||
out += content;
|
||||
prev = line;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const JUNK_TITLES = /^(untitled|powerpoint presentation|slide 1|document\d*|microsoft word|title)$/i;
|
||||
|
||||
export function tidyTitle(raw: unknown): string | undefined {
|
||||
if (typeof raw !== 'string') return undefined;
|
||||
let title = cleanText(raw).replace(/\s+/g, ' ').trim();
|
||||
title = title.replace(/^Microsoft (Word|PowerPoint) - /i, '').replace(/\.(docx?|pptx?|pdf|rtf)$/i, '').trim();
|
||||
if (!title || title.length > 300 || JUNK_TITLES.test(title)) return undefined;
|
||||
return title;
|
||||
}
|
||||
|
||||
function errorName(error: unknown): string {
|
||||
return typeof error === 'object' && error !== null && 'name' in error ? String((error as { name: unknown }).name) : '';
|
||||
}
|
||||
|
||||
function toExtractionError(error: unknown): ExtractionError {
|
||||
if (error instanceof ExtractionError) return error;
|
||||
const name = errorName(error);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (name === 'PasswordException' || /password/i.test(message)) {
|
||||
return new ExtractionError('encrypted', 'This PDF is password-protected. Remove the password and upload it again.', { cause: error });
|
||||
}
|
||||
if (
|
||||
name === 'InvalidPDFException' ||
|
||||
name === 'FormatError' ||
|
||||
name === 'XRefParseException' ||
|
||||
name === 'XRefEntryException' ||
|
||||
name === 'MissingDataException' ||
|
||||
/invalid pdf|xref|bad .*(trailer|catalog)|invalid root/i.test(message)
|
||||
) {
|
||||
return new ExtractionError('corrupt', 'This PDF looks damaged and could not be opened. Try exporting or downloading it again.', { cause: error });
|
||||
}
|
||||
return new ExtractionError('failed', "We couldn't read this PDF. Try exporting it again, or upload a different copy.", { cause: error });
|
||||
}
|
||||
|
||||
function cancelled(signal: AbortSignal): ExtractionError {
|
||||
return new ExtractionError('failed', 'Reading this file was cancelled.', { cause: signal.reason });
|
||||
}
|
||||
|
||||
export async function extractPdf(bytes: Uint8Array, ctx: PdfContext = {}): Promise<ExtractResult> {
|
||||
const { onProgress, signal } = ctx;
|
||||
if (signal?.aborted) throw cancelled(signal);
|
||||
|
||||
let pdf: Awaited<ReturnType<typeof getDocumentProxy>>;
|
||||
try {
|
||||
// pdf.js takes ownership of (transfers) the buffer it is given, so hand it a private copy.
|
||||
const options: Record<string, unknown> = { verbosity: 0, isEvalSupported: false, disableFontFace: true, useSystemFonts: false, stopAtErrors: false };
|
||||
pdf = await getDocumentProxy(bytes.slice(), options);
|
||||
} catch (error) {
|
||||
throw toExtractionError(error);
|
||||
}
|
||||
|
||||
try {
|
||||
const total = pdf.numPages;
|
||||
if (!Number.isFinite(total) || total < 1) {
|
||||
throw new ExtractionError('corrupt', "This PDF doesn't contain any pages.");
|
||||
}
|
||||
|
||||
let title: string | undefined;
|
||||
try {
|
||||
const meta = await pdf.getMetadata();
|
||||
const info = (meta?.info ?? {}) as Record<string, unknown>;
|
||||
title = tidyTitle(info.Title);
|
||||
if (!title && meta?.metadata) title = tidyTitle(meta.metadata.get('dc:title'));
|
||||
} catch {
|
||||
// Metadata is optional.
|
||||
}
|
||||
|
||||
const pageTexts: string[] = new Array(total).fill('');
|
||||
let failedPages = 0;
|
||||
let lastYield = Date.now();
|
||||
for (let n = 1; n <= total; n++) {
|
||||
if (signal?.aborted) throw cancelled(signal);
|
||||
try {
|
||||
const page = await pdf.getPage(n);
|
||||
try {
|
||||
const content = await page.getTextContent();
|
||||
pageTexts[n - 1] = textItemsToText(content.items as TextItemLike[]);
|
||||
} finally {
|
||||
page.cleanup();
|
||||
}
|
||||
} catch (error) {
|
||||
if (errorName(error) === 'PasswordException') throw error;
|
||||
failedPages++;
|
||||
}
|
||||
if (n % CLEANUP_EVERY === 0) {
|
||||
try {
|
||||
await pdf.cleanup();
|
||||
} catch {
|
||||
// Best effort.
|
||||
}
|
||||
}
|
||||
onProgress?.(n, total);
|
||||
// pdf.js runs on this thread in Node: give the event loop a turn regularly.
|
||||
if (Date.now() - lastYield > 20) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
lastYield = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
if (failedPages === total) {
|
||||
throw new ExtractionError('corrupt', 'This PDF looks damaged and none of its pages could be read. Try exporting or downloading it again.');
|
||||
}
|
||||
|
||||
const cleaned = stripRepeatedPageLines(pageTexts.map((text) => cleanText(text))).map((text) => cleanText(text));
|
||||
const sections: ExtractedSection[] = [];
|
||||
let empty = 0;
|
||||
let chars = 0;
|
||||
for (let i = 0; i < total; i++) {
|
||||
const text = cleaned[i];
|
||||
if (!text) {
|
||||
empty++;
|
||||
continue;
|
||||
}
|
||||
chars += text.length;
|
||||
sections.push({ text, page: i + 1 });
|
||||
}
|
||||
|
||||
const warnings: string[] = [];
|
||||
if (empty > 0) {
|
||||
warnings.push(
|
||||
empty === total
|
||||
? `None of the ${total} page${total === 1 ? '' : 's'} had extractable text - this looks like a scanned PDF.`
|
||||
: `${empty} of ${total} pages had no extractable text.`,
|
||||
);
|
||||
}
|
||||
if (failedPages > 0) warnings.push(`${failedPages} of ${total} pages could not be read.`);
|
||||
|
||||
const result: ExtractResult = {
|
||||
kind: 'pdf',
|
||||
sections,
|
||||
pageCount: total,
|
||||
warnings,
|
||||
needsOcr: chars / total < OCR_CHARS_PER_PAGE,
|
||||
};
|
||||
if (title) result.title = title;
|
||||
return result;
|
||||
} catch (error) {
|
||||
throw toExtractionError(error);
|
||||
} finally {
|
||||
try {
|
||||
await pdf.loadingTask.destroy();
|
||||
} catch {
|
||||
// Nothing useful to do.
|
||||
}
|
||||
}
|
||||
}
|
||||
424
server/documents/pptx.ts
Normal file
424
server/documents/pptx.ts
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { cleanText } from './chunk.ts';
|
||||
import { decodeEntities, renderTable } from './markup.ts';
|
||||
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
||||
import { ZipReader, type ZipLimits } from './zip.ts';
|
||||
|
||||
export interface OfficeContext {
|
||||
limits: ZipLimits;
|
||||
onProgress?: (done: number, total: number) => void;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
type XmlNode = Record<string, unknown>;
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '',
|
||||
preserveOrder: true,
|
||||
trimValues: false,
|
||||
parseTagValue: false,
|
||||
parseAttributeValue: false,
|
||||
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
|
||||
processEntities: false,
|
||||
ignoreDeclaration: true,
|
||||
ignorePiTags: true,
|
||||
});
|
||||
|
||||
function parseXml(xml: string): XmlNode[] {
|
||||
const parsed = parser.parse(xml) as unknown;
|
||||
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
|
||||
}
|
||||
|
||||
function tagOf(node: XmlNode): string {
|
||||
for (const key in node) if (key !== ':@') return key;
|
||||
return '';
|
||||
}
|
||||
|
||||
/** Element name without its namespace prefix. */
|
||||
function localName(node: XmlNode): string {
|
||||
const tag = tagOf(node);
|
||||
const colon = tag.indexOf(':');
|
||||
return colon < 0 ? tag : tag.slice(colon + 1);
|
||||
}
|
||||
|
||||
function childrenOf(node: XmlNode): XmlNode[] {
|
||||
const value = node[tagOf(node)];
|
||||
return Array.isArray(value) ? (value as XmlNode[]) : [];
|
||||
}
|
||||
|
||||
function attr(node: XmlNode, name: string): string | undefined {
|
||||
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
||||
if (!attrs) return undefined;
|
||||
const direct = attrs[name];
|
||||
if (typeof direct === 'string') return direct;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
|
||||
function attrLocal(node: XmlNode, local: string): string | undefined {
|
||||
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
||||
if (!attrs) return undefined;
|
||||
for (const key in attrs) {
|
||||
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function child(node: XmlNode, local: string): XmlNode | undefined {
|
||||
for (const c of childrenOf(node)) if (localName(c) === local) return c;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
|
||||
if (depth > 40) return undefined;
|
||||
for (const node of nodes) {
|
||||
if (localName(node) === local) return node;
|
||||
const found = findDeep(childrenOf(node), local, depth + 1);
|
||||
if (found) return found;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Concatenated text of every a:t / m:t descendant. */
|
||||
function deepText(node: XmlNode, depth = 0): string {
|
||||
if (depth > 40) return '';
|
||||
const name = localName(node);
|
||||
if (name === '#text') return '';
|
||||
if (name === 't') {
|
||||
let out = '';
|
||||
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
|
||||
return decodeEntities(out);
|
||||
}
|
||||
let out = '';
|
||||
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
|
||||
return out;
|
||||
}
|
||||
|
||||
interface Paragraph {
|
||||
text: string;
|
||||
level: number;
|
||||
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
|
||||
bullet: boolean | undefined;
|
||||
}
|
||||
|
||||
function readParagraph(p: XmlNode): Paragraph {
|
||||
let text = '';
|
||||
let level = 0;
|
||||
let bullet: boolean | undefined;
|
||||
for (const c of childrenOf(p)) {
|
||||
switch (localName(c)) {
|
||||
case 'pPr': {
|
||||
const lvl = Number(attr(c, 'lvl'));
|
||||
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
|
||||
for (const b of childrenOf(c)) {
|
||||
const name = localName(b);
|
||||
if (name === 'buNone') bullet = false;
|
||||
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'br':
|
||||
text += '\n';
|
||||
break;
|
||||
case 'fld': {
|
||||
const type = attr(c, 'type') ?? '';
|
||||
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
|
||||
break;
|
||||
}
|
||||
case 'endParaRPr':
|
||||
case '#text':
|
||||
break;
|
||||
default:
|
||||
text += deepText(c);
|
||||
}
|
||||
}
|
||||
text = text
|
||||
.replace(/[\u000B\u2028]/g, '\n')
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
|
||||
.filter((line) => line.length > 0)
|
||||
.join('\n');
|
||||
return { text, level, bullet };
|
||||
}
|
||||
|
||||
function readParagraphs(txBody: XmlNode): Paragraph[] {
|
||||
const out: Paragraph[] = [];
|
||||
for (const c of childrenOf(txBody)) {
|
||||
if (localName(c) !== 'p') continue;
|
||||
const paragraph = readParagraph(c);
|
||||
if (paragraph.text) out.push(paragraph);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
|
||||
const lines: string[] = [];
|
||||
for (const p of paragraphs) {
|
||||
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
|
||||
if (!bulleted) {
|
||||
lines.push(p.text);
|
||||
continue;
|
||||
}
|
||||
const indent = ' '.repeat(p.level);
|
||||
const parts = p.text.split('\n');
|
||||
lines.push(`${indent}- ${parts[0]}`);
|
||||
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function readTable(tbl: XmlNode): string {
|
||||
const rows: string[][] = [];
|
||||
for (const tr of childrenOf(tbl)) {
|
||||
if (localName(tr) !== 'tr') continue;
|
||||
const cells: string[] = [];
|
||||
for (const tc of childrenOf(tr)) {
|
||||
if (localName(tc) !== 'tc') continue;
|
||||
const txBody = child(tc, 'txBody');
|
||||
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
|
||||
}
|
||||
rows.push(cells);
|
||||
}
|
||||
return renderTable(rows);
|
||||
}
|
||||
|
||||
interface SlideContent {
|
||||
title: string;
|
||||
blocks: string[];
|
||||
}
|
||||
|
||||
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
|
||||
|
||||
function placeholderType(shape: XmlNode): string | null {
|
||||
const nv = child(shape, 'nvSpPr');
|
||||
const nvPr = nv ? child(nv, 'nvPr') : undefined;
|
||||
const ph = nvPr ? child(nvPr, 'ph') : undefined;
|
||||
if (!ph) return null;
|
||||
return attr(ph, 'type') ?? 'body';
|
||||
}
|
||||
|
||||
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
|
||||
if (depth > 30) return;
|
||||
for (const node of nodes) {
|
||||
switch (localName(node)) {
|
||||
case 'sp': {
|
||||
const type = placeholderType(node);
|
||||
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
|
||||
const txBody = child(node, 'txBody');
|
||||
if (!txBody) break;
|
||||
const paragraphs = readParagraphs(txBody);
|
||||
if (paragraphs.length === 0) break;
|
||||
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
|
||||
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
|
||||
} else {
|
||||
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
|
||||
const text = renderParagraphs(paragraphs, bullets);
|
||||
if (text) into.blocks.push(text);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'grpSp':
|
||||
walkShapes(childrenOf(node), into, depth + 1);
|
||||
break;
|
||||
case 'graphicFrame': {
|
||||
const tbl = findDeep(childrenOf(node), 'tbl');
|
||||
if (tbl) {
|
||||
const table = readTable(tbl);
|
||||
if (table) into.blocks.push(table);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'AlternateContent': {
|
||||
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
|
||||
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readSlide(xml: string): SlideContent {
|
||||
const content: SlideContent = { title: '', blocks: [] };
|
||||
const spTree = findDeep(parseXml(xml), 'spTree');
|
||||
if (spTree) walkShapes(childrenOf(spTree), content, 0);
|
||||
return content;
|
||||
}
|
||||
|
||||
function readNotes(xml: string): string {
|
||||
const spTree = findDeep(parseXml(xml), 'spTree');
|
||||
if (!spTree) return '';
|
||||
const parts: string[] = [];
|
||||
for (const shape of childrenOf(spTree)) {
|
||||
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
|
||||
const txBody = child(shape, 'txBody');
|
||||
if (!txBody) continue;
|
||||
const text = renderParagraphs(readParagraphs(txBody), false);
|
||||
if (text) parts.push(text);
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
interface Relationship {
|
||||
id: string;
|
||||
type: string;
|
||||
target: string;
|
||||
external: boolean;
|
||||
}
|
||||
|
||||
function readRelationships(xml: string | null): Relationship[] {
|
||||
if (!xml) return [];
|
||||
const out: Relationship[] = [];
|
||||
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = tags.exec(xml)) !== null) {
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
||||
let pair: RegExpExecArray | null;
|
||||
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
|
||||
if (attrs.Id && attrs.Target) {
|
||||
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Resolves a relationship target against the directory of the part that owns it. */
|
||||
function resolveTarget(baseDir: string, target: string): string {
|
||||
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
|
||||
for (const segment of target.split('/')) {
|
||||
if (!segment || segment === '.') continue;
|
||||
if (segment === '..') parts.pop();
|
||||
else parts.push(segment);
|
||||
}
|
||||
return parts.join('/');
|
||||
}
|
||||
|
||||
function relsPathFor(part: string): string {
|
||||
const slash = part.lastIndexOf('/');
|
||||
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
|
||||
}
|
||||
|
||||
function slideOrder(zip: ZipReader): string[] {
|
||||
const ordered: string[] = [];
|
||||
const presentation = zip.readText('ppt/presentation.xml');
|
||||
if (presentation) {
|
||||
const targets = new Map<string, string>();
|
||||
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
|
||||
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
|
||||
}
|
||||
const list = findDeep(parseXml(presentation), 'sldIdLst');
|
||||
if (list) {
|
||||
for (const item of childrenOf(list)) {
|
||||
if (localName(item) !== 'sldId') continue;
|
||||
const id = attrLocal(item, 'id');
|
||||
const path = id ? targets.get(id) : undefined;
|
||||
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ordered.length > 0) return ordered;
|
||||
// Damaged or unusual package: fall back to numeric file order.
|
||||
return zip
|
||||
.names()
|
||||
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
|
||||
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
|
||||
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
|
||||
.map((item) => item.name);
|
||||
}
|
||||
|
||||
export function readCoreTitle(zip: ZipReader): string | undefined {
|
||||
let xml: string | null = null;
|
||||
try {
|
||||
xml = zip.readText('docProps/core.xml');
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (!xml) return undefined;
|
||||
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
|
||||
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
|
||||
return title || undefined;
|
||||
}
|
||||
|
||||
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
||||
const zip = new ZipReader(bytes, ctx.limits);
|
||||
const slides = slideOrder(zip);
|
||||
if (slides.length === 0) {
|
||||
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
|
||||
}
|
||||
|
||||
const sections: ExtractedSection[] = [];
|
||||
const warnings: string[] = [];
|
||||
let withoutText = 0;
|
||||
let unreadable = 0;
|
||||
let totalChars = 0;
|
||||
let lastYield = Date.now();
|
||||
|
||||
for (let i = 0; i < slides.length; i++) {
|
||||
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
||||
const path = slides[i];
|
||||
let text = '';
|
||||
let title = '';
|
||||
try {
|
||||
const xml = zip.readText(path);
|
||||
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
|
||||
title = slide.title;
|
||||
const parts: string[] = [];
|
||||
if (slide.title) parts.push(`## ${slide.title}`);
|
||||
parts.push(...slide.blocks);
|
||||
|
||||
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
|
||||
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
|
||||
);
|
||||
if (notesRel) {
|
||||
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
|
||||
const notes = notesXml ? readNotes(notesXml) : '';
|
||||
if (notes) parts.push(`Notes:\n${notes}`);
|
||||
}
|
||||
text = cleanText(parts.join('\n\n'));
|
||||
} catch (error) {
|
||||
if (error instanceof ExtractionError) throw error;
|
||||
if ((error as { code?: string })?.code === 'too_large') throw error;
|
||||
unreadable++;
|
||||
}
|
||||
|
||||
if (text) {
|
||||
totalChars += text.length;
|
||||
const section: ExtractedSection = { text, page: i + 1 };
|
||||
if (title) section.heading = title;
|
||||
sections.push(section);
|
||||
} else {
|
||||
withoutText++;
|
||||
}
|
||||
ctx.onProgress?.(i + 1, slides.length);
|
||||
if (Date.now() - lastYield > 20) {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
lastYield = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
const total = slides.length;
|
||||
if (unreadable === total) {
|
||||
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
|
||||
}
|
||||
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
|
||||
if (withoutText === total) {
|
||||
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
|
||||
} else if (withoutText * 2 > total) {
|
||||
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
|
||||
}
|
||||
|
||||
const result: ExtractResult = {
|
||||
kind: 'pptx',
|
||||
sections,
|
||||
pageCount: total,
|
||||
warnings,
|
||||
needsOcr: totalChars < total * 25,
|
||||
};
|
||||
const title = readCoreTitle(zip);
|
||||
if (title) result.title = title;
|
||||
return result;
|
||||
}
|
||||
270
server/documents/testFixtures.ts
Normal file
270
server/documents/testFixtures.ts
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
|
||||
import { strToU8, zipSync } from 'fflate';
|
||||
|
||||
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
|
||||
|
||||
function pdfString(text: string): string {
|
||||
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
|
||||
}
|
||||
|
||||
export interface PdfPage {
|
||||
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
|
||||
lines?: string[];
|
||||
/** Raw content stream, used instead of `lines`. */
|
||||
raw?: string;
|
||||
}
|
||||
|
||||
export interface PdfOptions {
|
||||
title?: string;
|
||||
/** Adds a standard security handler whose user password is not empty. */
|
||||
encrypted?: boolean;
|
||||
}
|
||||
|
||||
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
|
||||
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
|
||||
const objects: string[] = [];
|
||||
const add = (body: string): number => {
|
||||
objects.push(body);
|
||||
return objects.length;
|
||||
};
|
||||
const catalogId = add('');
|
||||
const pagesId = add('');
|
||||
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
|
||||
|
||||
const pageIds: number[] = [];
|
||||
for (const page of pages) {
|
||||
let stream = page.raw ?? '';
|
||||
if (page.raw === undefined && page.lines && page.lines.length > 0) {
|
||||
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
|
||||
for (const line of page.lines) {
|
||||
if (line === '') ops.push('0 -14 Td');
|
||||
else ops.push(`${pdfString(line)} Tj`, 'T*');
|
||||
}
|
||||
ops.push('ET');
|
||||
stream = ops.join('\n');
|
||||
}
|
||||
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
|
||||
pageIds.push(
|
||||
add(
|
||||
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
|
||||
),
|
||||
);
|
||||
}
|
||||
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
|
||||
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
|
||||
|
||||
let infoId = 0;
|
||||
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
|
||||
let encryptId = 0;
|
||||
if (options.encrypted) {
|
||||
const o = 'a1'.repeat(32);
|
||||
const u = 'b2'.repeat(32);
|
||||
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
|
||||
}
|
||||
|
||||
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
|
||||
const offsets: number[] = [];
|
||||
objects.forEach((body, index) => {
|
||||
offsets.push(out.length);
|
||||
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
|
||||
});
|
||||
const xrefOffset = out.length;
|
||||
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
||||
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
|
||||
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
|
||||
if (infoId) out += ` /Info ${infoId} 0 R`;
|
||||
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
|
||||
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
|
||||
return latin1(out);
|
||||
}
|
||||
|
||||
const xmlEscape = (text: string): string => text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
||||
|
||||
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
|
||||
|
||||
function coreProps(title: string | undefined): string {
|
||||
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
|
||||
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
|
||||
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DOCX
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type DocxBlock =
|
||||
| { type: 'heading'; level: number; text: string }
|
||||
| { type: 'p'; text: string; bold?: string }
|
||||
| { type: 'bullet'; text: string; level?: number }
|
||||
| { type: 'number'; text: string; level?: number }
|
||||
| { type: 'table'; rows: string[][] }
|
||||
| { type: 'image' };
|
||||
|
||||
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
|
||||
|
||||
function docxRun(text: string, bold = false): string {
|
||||
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
|
||||
}
|
||||
|
||||
function docxBlock(block: DocxBlock): string {
|
||||
switch (block.type) {
|
||||
case 'heading':
|
||||
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
|
||||
case 'p':
|
||||
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
|
||||
case 'bullet':
|
||||
case 'number':
|
||||
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
|
||||
case 'table':
|
||||
return `<w:tbl>${block.rows
|
||||
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
|
||||
.join('')}</w:tbl>`;
|
||||
case 'image':
|
||||
return '<w:p/>';
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
|
||||
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
|
||||
const levels = (format: string) =>
|
||||
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
|
||||
const files: Record<string, Uint8Array> = {
|
||||
'[Content_Types].xml': strToU8(
|
||||
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
|
||||
),
|
||||
'_rels/.rels': strToU8(
|
||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
|
||||
),
|
||||
'word/_rels/document.xml.rels': strToU8(
|
||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
|
||||
),
|
||||
'word/numbering.xml': strToU8(
|
||||
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
|
||||
),
|
||||
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
|
||||
'docProps/core.xml': strToU8(coreProps(options.title)),
|
||||
...options.extraFiles,
|
||||
};
|
||||
return zipSync(files);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PPTX
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface PptxSlide {
|
||||
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
|
||||
file: number;
|
||||
title?: string;
|
||||
/** Body placeholder paragraphs. */
|
||||
bullets?: Array<string | { text: string; level: number }>;
|
||||
/** A free text box (not a placeholder). */
|
||||
textBox?: string[];
|
||||
table?: string[][];
|
||||
notes?: string[];
|
||||
}
|
||||
|
||||
const P_NS =
|
||||
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
|
||||
|
||||
function aParagraph(text: string, level = 0): string {
|
||||
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
|
||||
}
|
||||
|
||||
function shape(id: number, placeholder: string | null, paragraphs: string): string {
|
||||
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
|
||||
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
|
||||
}
|
||||
|
||||
function slideXml(slide: PptxSlide): string {
|
||||
const shapes: string[] = [];
|
||||
let id = 2;
|
||||
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
|
||||
if (slide.bullets) {
|
||||
shapes.push(
|
||||
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
|
||||
);
|
||||
}
|
||||
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
|
||||
if (slide.table) {
|
||||
const rows = slide.table
|
||||
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
|
||||
.join('');
|
||||
shapes.push(
|
||||
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
|
||||
);
|
||||
}
|
||||
// Every slide also carries a slide-number placeholder, which must be ignored.
|
||||
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
|
||||
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
|
||||
}
|
||||
|
||||
function notesXml(lines: string[]): string {
|
||||
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
|
||||
2,
|
||||
'sldImg',
|
||||
'',
|
||||
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
|
||||
}
|
||||
|
||||
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
|
||||
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
|
||||
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
|
||||
const files: Record<string, Uint8Array> = {
|
||||
'[Content_Types].xml': strToU8(
|
||||
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
|
||||
),
|
||||
'_rels/.rels': strToU8(
|
||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
|
||||
),
|
||||
'docProps/core.xml': strToU8(coreProps(options.title)),
|
||||
'ppt/presentation.xml': strToU8(
|
||||
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
|
||||
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
|
||||
.join('')}</p:sldIdLst></p:presentation>`,
|
||||
),
|
||||
'ppt/_rels/presentation.xml.rels': strToU8(
|
||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
|
||||
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
|
||||
.join('')}</Relationships>`,
|
||||
),
|
||||
};
|
||||
for (const slide of slides) {
|
||||
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
|
||||
if (slide.notes) {
|
||||
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
|
||||
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
|
||||
);
|
||||
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
|
||||
}
|
||||
}
|
||||
return zipSync(files);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Misc
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
|
||||
export const TINY_PNG = Uint8Array.from([
|
||||
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
|
||||
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
|
||||
]);
|
||||
|
||||
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
|
||||
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
|
||||
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
|
||||
|
||||
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
|
||||
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
|
||||
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
|
||||
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
|
||||
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
|
||||
for (let i = 0; i + 46 < zip.length; i++) {
|
||||
if (view.getUint32(i, true) !== 0x02014b50) continue;
|
||||
const nameLen = view.getUint16(i + 28, true);
|
||||
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
|
||||
if (entryName === name) view.setUint32(i + 24, 16, true);
|
||||
}
|
||||
return zip;
|
||||
}
|
||||
53
server/documents/types.ts
Normal file
53
server/documents/types.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/** Contract for the document extraction + chunking pipeline. */
|
||||
|
||||
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
|
||||
|
||||
export interface ExtractedSection {
|
||||
/** Plain text (Markdown allowed) for this page / slide / section. */
|
||||
text: string;
|
||||
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
|
||||
page?: number;
|
||||
/** Nearest heading or slide title, when known. */
|
||||
heading?: string;
|
||||
}
|
||||
|
||||
export interface ExtractResult {
|
||||
kind: DocKind;
|
||||
sections: ExtractedSection[];
|
||||
/** Page or slide count for paged formats. */
|
||||
pageCount?: number;
|
||||
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
|
||||
warnings: string[];
|
||||
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
|
||||
needsOcr: boolean;
|
||||
/** Document title from metadata, when present. */
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface DocChunk {
|
||||
seq: number;
|
||||
text: string;
|
||||
heading: string;
|
||||
pageStart: number | null;
|
||||
pageEnd: number | null;
|
||||
tokens: number;
|
||||
}
|
||||
|
||||
export interface ChunkOptions {
|
||||
/** Preferred chunk size in estimated tokens. */
|
||||
targetTokens?: number;
|
||||
/** Hard ceiling; a chunk is split before exceeding it. */
|
||||
maxTokens?: number;
|
||||
/** Tokens of trailing context repeated at the start of the next chunk. */
|
||||
overlapTokens?: number;
|
||||
}
|
||||
|
||||
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
|
||||
export class ExtractionError extends Error {
|
||||
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
|
||||
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
|
||||
super(message, opts);
|
||||
this.name = 'ExtractionError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
213
server/documents/zip.ts
Normal file
213
server/documents/zip.ts
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
/**
|
||||
* Minimal, defensive ZIP reader for OOXML packages (DOCX / PPTX).
|
||||
*
|
||||
* Reads the central directory itself and inflates entries through fflate's
|
||||
* streaming inflater in small input slices, so both memory and CPU are bounded
|
||||
* by the limits below even when the archive lies about its sizes (zip bombs).
|
||||
*/
|
||||
import { Inflate } from 'fflate';
|
||||
|
||||
export interface ZipEntry {
|
||||
name: string;
|
||||
method: number;
|
||||
compressedSize: number;
|
||||
size: number;
|
||||
offset: number;
|
||||
encrypted: boolean;
|
||||
}
|
||||
|
||||
export interface ZipLimits {
|
||||
/** Largest uncompressed size accepted for a single entry. */
|
||||
maxEntryBytes: number;
|
||||
/** Largest total of uncompressed bytes read through one reader. */
|
||||
maxTotalBytes: number;
|
||||
}
|
||||
|
||||
export class ZipError extends Error {
|
||||
code: 'corrupt' | 'too_large';
|
||||
constructor(code: ZipError['code'], message: string) {
|
||||
super(message);
|
||||
this.name = 'ZipError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
const EOCD_SIG = 0x06054b50;
|
||||
const ZIP64_LOCATOR_SIG = 0x07064b50;
|
||||
const ZIP64_EOCD_SIG = 0x06064b50;
|
||||
const CENTRAL_SIG = 0x02014b50;
|
||||
const LOCAL_SIG = 0x04034b50;
|
||||
const MAX_ENTRIES = 20_000;
|
||||
const INFLATE_STEP = 64 * 1024;
|
||||
|
||||
const nameDecoder = new TextDecoder('utf-8');
|
||||
|
||||
function u16(b: Uint8Array, o: number): number {
|
||||
return b[o] | (b[o + 1] << 8);
|
||||
}
|
||||
|
||||
function u32(b: Uint8Array, o: number): number {
|
||||
return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
function u64(b: Uint8Array, o: number): number {
|
||||
return u32(b, o) + u32(b, o + 4) * 0x1_0000_0000;
|
||||
}
|
||||
|
||||
/** Parses the central directory. Throws ZipError('corrupt') when it cannot be read. */
|
||||
export function listZip(bytes: Uint8Array): ZipEntry[] {
|
||||
const len = bytes.length;
|
||||
if (len < 22) throw new ZipError('corrupt', 'Not a zip archive');
|
||||
let eocd = -1;
|
||||
const stop = Math.max(0, len - 22 - 0xffff);
|
||||
for (let i = len - 22; i >= stop; i--) {
|
||||
if (bytes[i] === 0x50 && u32(bytes, i) === EOCD_SIG) {
|
||||
eocd = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (eocd < 0) throw new ZipError('corrupt', 'Zip directory not found');
|
||||
|
||||
let count = u16(bytes, eocd + 10);
|
||||
let dirOffset = u32(bytes, eocd + 16);
|
||||
if (count === 0xffff || dirOffset === 0xffffffff) {
|
||||
const locator = eocd - 20;
|
||||
if (locator < 0 || u32(bytes, locator) !== ZIP64_LOCATOR_SIG) throw new ZipError('corrupt', 'Bad zip64 locator');
|
||||
const z = u64(bytes, locator + 8);
|
||||
if (z + 56 > len || u32(bytes, z) !== ZIP64_EOCD_SIG) throw new ZipError('corrupt', 'Bad zip64 directory');
|
||||
count = u64(bytes, z + 32);
|
||||
dirOffset = u64(bytes, z + 48);
|
||||
}
|
||||
if (count > MAX_ENTRIES) throw new ZipError('too_large', 'Too many files in archive');
|
||||
|
||||
const entries: ZipEntry[] = [];
|
||||
let p = dirOffset;
|
||||
for (let i = 0; i < count; i++) {
|
||||
if (p + 46 > len || u32(bytes, p) !== CENTRAL_SIG) throw new ZipError('corrupt', 'Bad zip directory entry');
|
||||
const flags = u16(bytes, p + 8);
|
||||
const method = u16(bytes, p + 10);
|
||||
let compressedSize = u32(bytes, p + 20);
|
||||
let size = u32(bytes, p + 24);
|
||||
const nameLen = u16(bytes, p + 28);
|
||||
const extraLen = u16(bytes, p + 30);
|
||||
const commentLen = u16(bytes, p + 32);
|
||||
let offset = u32(bytes, p + 42);
|
||||
const nameStart = p + 46;
|
||||
const extraStart = nameStart + nameLen;
|
||||
const next = extraStart + extraLen + commentLen;
|
||||
if (next > len) throw new ZipError('corrupt', 'Truncated zip directory');
|
||||
if (size === 0xffffffff || compressedSize === 0xffffffff || offset === 0xffffffff) {
|
||||
// zip64 extended information: only the saturated fields are present, in this order.
|
||||
let e = extraStart;
|
||||
const extraEnd = extraStart + extraLen;
|
||||
while (e + 4 <= extraEnd) {
|
||||
const id = u16(bytes, e);
|
||||
const dataLen = u16(bytes, e + 2);
|
||||
if (id === 1) {
|
||||
let q = e + 4;
|
||||
if (size === 0xffffffff && q + 8 <= extraEnd) (size = u64(bytes, q)), (q += 8);
|
||||
if (compressedSize === 0xffffffff && q + 8 <= extraEnd) (compressedSize = u64(bytes, q)), (q += 8);
|
||||
if (offset === 0xffffffff && q + 8 <= extraEnd) offset = u64(bytes, q);
|
||||
break;
|
||||
}
|
||||
e += 4 + dataLen;
|
||||
}
|
||||
}
|
||||
const name = nameDecoder.decode(bytes.subarray(nameStart, extraStart)).replace(/\\/g, '/');
|
||||
entries.push({ name, method, compressedSize, size, offset, encrypted: (flags & 1) === 1 });
|
||||
p = next;
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** Random-access reader over a zip held in memory, with per-entry and total size limits. */
|
||||
export class ZipReader {
|
||||
private readonly bytes: Uint8Array;
|
||||
private readonly limits: ZipLimits;
|
||||
private readonly byName = new Map<string, ZipEntry>();
|
||||
private totalRead = 0;
|
||||
readonly entries: ZipEntry[];
|
||||
|
||||
constructor(bytes: Uint8Array, limits: ZipLimits) {
|
||||
this.bytes = bytes;
|
||||
this.limits = limits;
|
||||
this.entries = listZip(bytes);
|
||||
for (const entry of this.entries) if (!this.byName.has(entry.name)) this.byName.set(entry.name, entry);
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this.byName.has(name);
|
||||
}
|
||||
|
||||
names(): string[] {
|
||||
return this.entries.map((entry) => entry.name);
|
||||
}
|
||||
|
||||
/** Uncompressed bytes of an entry, or null when it does not exist. */
|
||||
read(name: string): Uint8Array | null {
|
||||
const entry = this.byName.get(name);
|
||||
if (!entry) return null;
|
||||
if (entry.encrypted) throw new ZipError('corrupt', 'Encrypted zip entry');
|
||||
if (entry.size > this.limits.maxEntryBytes) throw new ZipError('too_large', `Entry too large: ${name}`);
|
||||
if (this.totalRead + entry.size > this.limits.maxTotalBytes) throw new ZipError('too_large', 'Archive too large');
|
||||
|
||||
const b = this.bytes;
|
||||
const o = entry.offset;
|
||||
if (o + 30 > b.length || u32(b, o) !== LOCAL_SIG) throw new ZipError('corrupt', 'Bad zip file header');
|
||||
const dataStart = o + 30 + u16(b, o + 26) + u16(b, o + 28);
|
||||
const dataEnd = dataStart + entry.compressedSize;
|
||||
if (dataEnd > b.length) throw new ZipError('corrupt', 'Truncated zip entry');
|
||||
const compressed = b.subarray(dataStart, dataEnd);
|
||||
|
||||
let out: Uint8Array;
|
||||
if (entry.method === 0) {
|
||||
if (compressed.length !== entry.size) throw new ZipError('corrupt', 'Zip entry size mismatch');
|
||||
out = compressed;
|
||||
} else if (entry.method === 8) {
|
||||
out = this.inflate(compressed, entry.size);
|
||||
} else {
|
||||
throw new ZipError('corrupt', `Unsupported zip compression method ${entry.method}`);
|
||||
}
|
||||
this.totalRead += out.length;
|
||||
return out;
|
||||
}
|
||||
|
||||
readText(name: string): string | null {
|
||||
const data = this.read(name);
|
||||
if (!data) return null;
|
||||
let text = new TextDecoder('utf-8').decode(data);
|
||||
if (text.charCodeAt(0) === 0xfeff) text = text.slice(1);
|
||||
return text;
|
||||
}
|
||||
|
||||
private inflate(compressed: Uint8Array, expected: number): Uint8Array {
|
||||
const out = new Uint8Array(expected);
|
||||
let written = 0;
|
||||
let overflow = false;
|
||||
const inflater = new Inflate((chunk) => {
|
||||
if (overflow) return;
|
||||
if (written + chunk.length > expected) {
|
||||
overflow = true;
|
||||
return;
|
||||
}
|
||||
out.set(chunk, written);
|
||||
written += chunk.length;
|
||||
});
|
||||
try {
|
||||
if (compressed.length === 0) {
|
||||
inflater.push(compressed, true);
|
||||
} else {
|
||||
for (let p = 0; p < compressed.length && !overflow; p += INFLATE_STEP) {
|
||||
const end = Math.min(p + INFLATE_STEP, compressed.length);
|
||||
inflater.push(compressed.subarray(p, end), end === compressed.length);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
throw new ZipError('corrupt', 'Zip entry could not be decompressed');
|
||||
}
|
||||
// More data than the directory declared is the signature of a zip bomb.
|
||||
if (overflow) throw new ZipError('too_large', 'Zip entry larger than declared');
|
||||
if (written !== expected) throw new ZipError('corrupt', 'Zip entry size mismatch');
|
||||
return out;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue