Study/server/documents/documents.test.ts
Elijah 3e74ecd843 Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
2026-11-11 08:16:41 -07:00

420 lines
18 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { strToU8, zipSync } from 'fflate';
import { describe, expect, it } from 'vitest';
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
import { detectKind, extractDocument } from './extract.ts';
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
import { ExtractionError, type ExtractedSection } from './types.ts';
const enc = (s: string) => new TextEncoder().encode(s);
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
async function codeOf(promise: Promise<unknown>): Promise<string> {
try {
await promise;
} catch (e) {
expect(e).toBeInstanceOf(ExtractionError);
return (e as ExtractionError).code;
}
throw new Error('expected rejection');
}
describe('detectKind', () => {
const pdf = buildPdf([{ lines: ['hello'] }]);
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
const pptx = buildPptx([{ file: 1, title: 'T' }]);
it('detects by magic bytes', () => {
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
});
it('trusts magic bytes over a wrong extension', () => {
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
});
it('uses the extension for text and markdown', () => {
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
});
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
});
});
describe('extractDocument: PDF', () => {
it('extracts multi-page text with page numbers, count and title', async () => {
const bytes = buildPdf(
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
{ title: 'Cardiology Notes' },
);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.kind).toBe('pdf');
expect(r.pageCount).toBe(2);
expect(r.title).toBe('Cardiology Notes');
expect(r.needsOcr).toBe(false);
expect(r.warnings).toEqual([]);
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
expect(r.sections[0].text).toContain('Heart failure overview');
expect(r.sections[1].text).toContain('Digoxin');
});
it('warns about a page without text but keeps page numbers', async () => {
const long = 'Readable text on this page, long enough to avoid OCR. ';
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
const r = await extractDocument({ bytes, filename: 'a.pdf' });
expect(r.pageCount).toBe(3);
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
});
it('flags an all-empty scanned PDF as needing OCR', async () => {
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
expect(r.needsOcr).toBe(true);
expect(r.sections).toEqual([]);
expect(r.pageCount).toBe(2);
expect(r.warnings.join(' ')).toMatch(/scanned/);
});
it('reports progress per page', async () => {
const seen: Array<[number, number]> = [];
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
expect(seen).toEqual([
[1, 2],
[2, 2],
]);
});
});
describe('extractDocument: DOCX', () => {
it('turns headings into sections and keeps lists and tables readable', async () => {
const bytes = buildDocx(
[
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
{ type: 'p', text: 'Intro paragraph about drugs.' },
{ type: 'bullet', text: 'Beta blockers' },
{ type: 'bullet', text: 'ACE inhibitors' },
{ type: 'heading', level: 2, text: 'Dosing' },
{ type: 'number', text: 'Check heart rate' },
{ type: 'number', text: 'Give the dose' },
{
type: 'table',
rows: [
['Drug', 'Dose'],
['Digoxin', '0.25 mg'],
],
},
],
{ title: 'Drug Guide' },
);
const r = await extractDocument({ bytes, filename: 'a.docx' });
expect(r.kind).toBe('docx');
expect(r.title).toBe('Drug Guide');
expect(r.needsOcr).toBe(false);
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
const first = r.sections[0].text;
expect(first).toContain('Intro paragraph');
expect(first).toContain('Beta blockers');
expect(first).toContain('ACE inhibitors');
const second = r.sections[1].text;
expect(second).toContain('Check heart rate');
expect(second).toContain('Digoxin');
expect(second).toContain('0.25 mg');
});
it('rejects a docx with no text as empty', async () => {
const bytes = buildDocx([{ type: 'p', text: '' }]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
});
});
describe('extractDocument: PPTX', () => {
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
const bytes = buildPptx(
[
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
{ file: 2 },
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
],
{ title: 'Deck Title' },
);
const r = await extractDocument({ bytes, filename: 'a.pptx' });
expect(r.kind).toBe('pptx');
expect(r.title).toBe('Deck Title');
expect(r.pageCount).toBe(4);
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
expect(r.sections[0].text).toContain('Welcome');
expect(r.sections[0].text).toContain('Sub point');
expect(r.sections[0].text).toContain('Say hello slowly');
expect(r.sections[1].text).toContain('Free text box content here');
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
});
it('reads tables', async () => {
const table = [
['A', 'B'],
['x', 'y'],
];
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
expect(allText(r)).toContain('x');
expect(allText(r)).toContain('B');
});
it('warns when most slides have no text', async () => {
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
});
});
describe('extractDocument: text, markdown, image', () => {
it('splits markdown by heading and ignores headings inside code fences', async () => {
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
expect(r.kind).toBe('markdown');
expect(r.title).toBe('Title');
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
expect(r.sections[1].text).toContain('# not a heading');
expect(r.needsOcr).toBe(false);
});
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
expect(r.kind).toBe('text');
expect(allText(r)).toContain('line two');
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
expect(allText(r2)).toBe('café');
expect(r2.warnings.length).toBe(1);
});
it('groups long plain text into several bounded sections', async () => {
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
expect(r.sections.length).toBeGreaterThan(1);
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
});
it('returns images with needsOcr and no sections', async () => {
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
});
});
describe('extractDocument: errors', () => {
it('rejects an empty file', async () => {
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
});
it('rejects whitespace-only text as empty', async () => {
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
});
it('rejects unsupported types and legacy .doc', async () => {
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
});
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
const marker = Buffer.from('EncryptedPackage', 'utf16le');
const bytes = Uint8Array.from([...OLE, ...marker]);
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
});
it('rejects corrupt PDFs', async () => {
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
});
it('rejects an encrypted PDF', async () => {
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
});
it('rejects a corrupt zip posing as docx/pptx', async () => {
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
const broken = good.slice(0, Math.floor(good.length / 2));
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a pptx zip that has no readable slides', async () => {
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
});
it('rejects a zip bomb whose headers lie about entry size', async () => {
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
expect(['too_large', 'corrupt']).toContain(code);
}, 30000);
it('honours an already-aborted signal', async () => {
const ac = new AbortController();
ac.abort();
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
});
});
describe('chunkSections', () => {
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
page: i + 1,
}));
const chunks = chunkSections(sections, opts);
expect(chunks.length).toBeGreaterThan(12);
chunks.forEach((c, i) => {
expect(c.seq).toBe(i);
expect(c.tokens).toBe(estimateTokens(c.text));
expect(c.tokens).toBeLessThanOrEqual(150);
});
});
it('packs small sections together with correct page range and first heading', () => {
const sections: ExtractedSection[] = [
{ text: 'Alpha text.', page: 1, heading: 'First' },
{ text: 'Beta text.', page: 2 },
{ text: 'Gamma text.', page: 3, heading: 'Third' },
];
const chunks = chunkSections(sections, opts);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
expect(chunks[0].text).toContain('Gamma');
});
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(c.pageStart).not.toBeNull();
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
}
expect(chunks[0].pageStart).toBe(1);
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
});
it('uses null pages and empty heading when sections have none', () => {
const [c] = chunkSections([{ text: 'No pages here.' }]);
expect(c.pageStart).toBeNull();
expect(c.pageEnd).toBeNull();
expect(c.heading).toBe('');
});
it('splits oversized sections without cutting words', () => {
const text = words(800, 'term');
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
expect(chunks.length).toBeGreaterThan(3);
const vocabulary = new Set(text.split(' '));
for (const c of chunks) {
expect(c.pageStart).toBe(5);
expect(c.pageEnd).toBe(5);
expect(c.heading).toBe('Big');
expect(c.tokens).toBeLessThanOrEqual(150);
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
}
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
expect(seen.size).toBe(800);
});
it('survives a giant unbroken string within the limit', () => {
const blob = 'x'.repeat(200_000);
const t0 = Date.now();
const chunks = chunkSections([{ text: blob }], opts);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
});
it('does not split surrogate pairs in unbroken emoji runs', () => {
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
expect(chunks.length).toBeGreaterThan(1);
for (const c of chunks) {
expect(wellFormed(c.text)).toBe(true);
expect(c.tokens).toBeLessThanOrEqual(150);
}
});
it('is deterministic', () => {
const sections = [
{ text: words(900), page: 1, heading: 'H' },
{ text: 'Short one.', page: 2 },
];
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
});
it('drops empty and whitespace-only sections', () => {
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
const chunks = chunkSections([
{ text: ' ', page: 1 },
{ text: 'Real content.', page: 2 },
{ text: '\n', page: 3 },
]);
expect(chunks).toHaveLength(1);
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
});
it('chunks about 2 MB of text quickly', () => {
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
const t0 = Date.now();
const chunks = chunkSections([{ text }]);
expect(Date.now() - t0).toBeLessThan(3000);
expect(chunks.length).toBeGreaterThan(100);
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
});
});
describe('cleanText', () => {
it('joins words hyphenated across a line break', () => {
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
});
it('keeps real compounds across a line break', () => {
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
});
it('collapses 3+ newlines to a paragraph break', () => {
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
expect(cleanText('a\n\nb')).toBe('a\n\nb');
});
it('strips control characters but keeps tabs and newlines', () => {
expect(cleanText('he\u0000l\u0007lo​ ­world\ttab\nnext')).toBe('hello world\ttab\nnext');
});
it('normalises CRLF and trims', () => {
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
expect(cleanText('')).toBe('');
});
it('is idempotent', () => {
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny  z');
expect(cleanText(once)).toBe(once);
});
});