import { strToU8, zipSync } from 'fflate'; import { describe, expect, it } from 'vitest'; import { chunkSections, cleanText, estimateTokens } from './chunk.ts'; import { detectKind, extractDocument } from './extract.ts'; import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts'; import { ExtractionError, type ExtractedSection } from './types.ts'; const enc = (s: string) => new TextEncoder().encode(s); const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n'); const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]); /** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */ const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s; async function codeOf(promise: Promise): Promise { try { await promise; } catch (e) { expect(e).toBeInstanceOf(ExtractionError); return (e as ExtractionError).code; } throw new Error('expected rejection'); } describe('detectKind', () => { const pdf = buildPdf([{ lines: ['hello'] }]); const docx = buildDocx([{ type: 'p', text: 'hi' }]); const pptx = buildPptx([{ file: 1, title: 'T' }]); it('detects by magic bytes', () => { expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf'); expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image'); expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image'); expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image'); expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image'); expect(detectKind('a.docx', undefined, docx)).toBe('docx'); expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx'); }); it('trusts magic bytes over a wrong extension', () => { expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf'); expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx'); expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image'); expect(detectKind('thing.bin', undefined, docx)).toBe('docx'); }); it('uses the extension for text and markdown', () => { expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text'); expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text'); expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown'); expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown'); expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown'); }); it('rejects unknown binary, legacy .doc, plain zips and empty input', () => { expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull(); expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull(); expect(detectKind('old.doc', undefined, OLE)).toBeNull(); expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull(); expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull(); }); }); describe('extractDocument: PDF', () => { it('extracts multi-page text with page numbers, count and title', async () => { const bytes = buildPdf( [{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }], { title: 'Cardiology Notes' }, ); const r = await extractDocument({ bytes, filename: 'a.pdf' }); expect(r.kind).toBe('pdf'); expect(r.pageCount).toBe(2); expect(r.title).toBe('Cardiology Notes'); expect(r.needsOcr).toBe(false); expect(r.warnings).toEqual([]); expect(r.sections.map((s) => s.page)).toEqual([1, 2]); expect(r.sections[0].text).toContain('Heart failure overview'); expect(r.sections[1].text).toContain('Digoxin'); }); it('warns about a page without text but keeps page numbers', async () => { const long = 'Readable text on this page, long enough to avoid OCR. '; const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]); const r = await extractDocument({ bytes, filename: 'a.pdf' }); expect(r.pageCount).toBe(3); expect(r.sections.map((s) => s.page)).toEqual([1, 3]); expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true); }); it('flags an all-empty scanned PDF as needing OCR', async () => { const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' }); expect(r.needsOcr).toBe(true); expect(r.sections).toEqual([]); expect(r.pageCount).toBe(2); expect(r.warnings.join(' ')).toMatch(/scanned/); }); it('reports progress per page', async () => { const seen: Array<[number, number]> = []; await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) }); expect(seen).toEqual([ [1, 2], [2, 2], ]); }); }); describe('extractDocument: DOCX', () => { it('turns headings into sections and keeps lists and tables readable', async () => { const bytes = buildDocx( [ { type: 'heading', level: 1, text: 'Cardiac Drugs' }, { type: 'p', text: 'Intro paragraph about drugs.' }, { type: 'bullet', text: 'Beta blockers' }, { type: 'bullet', text: 'ACE inhibitors' }, { type: 'heading', level: 2, text: 'Dosing' }, { type: 'number', text: 'Check heart rate' }, { type: 'number', text: 'Give the dose' }, { type: 'table', rows: [ ['Drug', 'Dose'], ['Digoxin', '0.25 mg'], ], }, ], { title: 'Drug Guide' }, ); const r = await extractDocument({ bytes, filename: 'a.docx' }); expect(r.kind).toBe('docx'); expect(r.title).toBe('Drug Guide'); expect(r.needsOcr).toBe(false); expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']); const first = r.sections[0].text; expect(first).toContain('Intro paragraph'); expect(first).toContain('Beta blockers'); expect(first).toContain('ACE inhibitors'); const second = r.sections[1].text; expect(second).toContain('Check heart rate'); expect(second).toContain('Digoxin'); expect(second).toContain('0.25 mg'); }); it('rejects a docx with no text as empty', async () => { const bytes = buildDocx([{ type: 'p', text: '' }]); expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty'); }); }); describe('extractDocument: PPTX', () => { it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => { const bytes = buildPptx( [ { file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] }, { file: 1, title: 'Middle', textBox: ['Free text box content here'] }, { file: 2 }, { file: 4, title: 'Closing', bullets: ['Thanks'] }, ], { title: 'Deck Title' }, ); const r = await extractDocument({ bytes, filename: 'a.pptx' }); expect(r.kind).toBe('pptx'); expect(r.title).toBe('Deck Title'); expect(r.pageCount).toBe(4); expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]); expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']); expect(r.sections[0].text).toContain('Welcome'); expect(r.sections[0].text).toContain('Sub point'); expect(r.sections[0].text).toContain('Say hello slowly'); expect(r.sections[1].text).toContain('Free text box content here'); expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored expect(r.warnings).toEqual([]); // one empty slide is not worth a warning }); it('reads tables', async () => { const table = [ ['A', 'B'], ['x', 'y'], ]; const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' }); expect(allText(r)).toContain('x'); expect(allText(r)).toContain('B'); }); it('warns when most slides have no text', async () => { const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' }); expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true); }); }); describe('extractDocument: text, markdown, image', () => { it('splits markdown by heading and ignores headings inside code fences', async () => { const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.'; const r = await extractDocument({ bytes: enc(md), filename: 'a.md' }); expect(r.kind).toBe('markdown'); expect(r.title).toBe('Title'); expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']); expect(r.sections[1].text).toContain('# not a heading'); expect(r.needsOcr).toBe(false); }); it('extracts plain text and falls back for non-UTF-8 with a warning', async () => { const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' }); expect(r.kind).toBe('text'); expect(allText(r)).toContain('line two'); const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' }); expect(allText(r2)).toBe('café'); expect(r2.warnings.length).toBe(1); }); it('groups long plain text into several bounded sections', async () => { const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n'); const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' }); expect(r.sections.length).toBeGreaterThan(1); for (const s of r.sections) expect(s.text.length).toBeLessThan(7000); }); it('returns images with needsOcr and no sections', async () => { const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' }); expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true }); }); }); describe('extractDocument: errors', () => { it('rejects an empty file', async () => { expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty'); }); it('rejects whitespace-only text as empty', async () => { expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty'); }); it('rejects unsupported types and legacy .doc', async () => { expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported'); await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/); expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported'); }); it('rejects a password-protected OOXML wrapper as encrypted', async () => { const marker = Buffer.from('EncryptedPackage', 'utf16le'); const bytes = Uint8Array.from([...OLE, ...marker]); expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted'); }); it('rejects corrupt PDFs', async () => { expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt'); }); it('rejects an encrypted PDF', async () => { const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true }); expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted'); }); it('rejects a corrupt zip posing as docx/pptx', async () => { const good = buildDocx([{ type: 'p', text: 'hello world' }]); const broken = good.slice(0, Math.floor(good.length / 2)); expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt'); expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt'); }); it('rejects a pptx zip that has no readable slides', async () => { const bytes = zipSync({ 'ppt/presentation.xml': strToU8('') }); expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt'); }); it('rejects a zip bomb whose headers lie about entry size', async () => { const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024); const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' })); expect(['too_large', 'corrupt']).toContain(code); }, 30000); it('honours an already-aborted signal', async () => { const ac = new AbortController(); ac.abort(); expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed'); }); }); describe('chunkSections', () => { const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' '); const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 }; it('keeps every chunk within maxTokens with contiguous 0-based seq', () => { const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({ text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '), page: i + 1, })); const chunks = chunkSections(sections, opts); expect(chunks.length).toBeGreaterThan(12); chunks.forEach((c, i) => { expect(c.seq).toBe(i); expect(c.tokens).toBe(estimateTokens(c.text)); expect(c.tokens).toBeLessThanOrEqual(150); }); }); it('packs small sections together with correct page range and first heading', () => { const sections: ExtractedSection[] = [ { text: 'Alpha text.', page: 1, heading: 'First' }, { text: 'Beta text.', page: 2 }, { text: 'Gamma text.', page: 3, heading: 'Third' }, ]; const chunks = chunkSections(sections, opts); expect(chunks).toHaveLength(1); expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 }); expect(chunks[0].text).toContain('Gamma'); }); it('starts a new chunk when packing would exceed the target and tracks ranges', () => { const big = (i: number) => ({ text: words(60, `s${i}x`), page: i }); const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 }); expect(chunks.length).toBeGreaterThan(1); for (const c of chunks) { expect(c.pageStart).not.toBeNull(); expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!); } expect(chunks[0].pageStart).toBe(1); expect(chunks[chunks.length - 1].pageEnd).toBe(4); }); it('uses null pages and empty heading when sections have none', () => { const [c] = chunkSections([{ text: 'No pages here.' }]); expect(c.pageStart).toBeNull(); expect(c.pageEnd).toBeNull(); expect(c.heading).toBe(''); }); it('splits oversized sections without cutting words', () => { const text = words(800, 'term'); const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts); expect(chunks.length).toBeGreaterThan(3); const vocabulary = new Set(text.split(' ')); for (const c of chunks) { expect(c.pageStart).toBe(5); expect(c.pageEnd).toBe(5); expect(c.heading).toBe('Big'); expect(c.tokens).toBeLessThanOrEqual(150); for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true); } const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/))); expect(seen.size).toBe(800); }); it('survives a giant unbroken string within the limit', () => { const blob = 'x'.repeat(200_000); const t0 = Date.now(); const chunks = chunkSections([{ text: blob }], opts); expect(Date.now() - t0).toBeLessThan(3000); expect(chunks.length).toBeGreaterThan(1); for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150); expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000); }); it('does not split surrogate pairs in unbroken emoji runs', () => { const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts); expect(chunks.length).toBeGreaterThan(1); for (const c of chunks) { expect(wellFormed(c.text)).toBe(true); expect(c.tokens).toBeLessThanOrEqual(150); } }); it('is deterministic', () => { const sections = [ { text: words(900), page: 1, heading: 'H' }, { text: 'Short one.', page: 2 }, ]; expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts)); }); it('drops empty and whitespace-only sections', () => { expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]); const chunks = chunkSections([ { text: ' ', page: 1 }, { text: 'Real content.', page: 2 }, { text: '\n', page: 3 }, ]); expect(chunks).toHaveLength(1); expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 }); }); it('chunks about 2 MB of text quickly', () => { const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n'; const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length)); const t0 = Date.now(); const chunks = chunkSections([{ text }]); expect(Date.now() - t0).toBeLessThan(3000); expect(chunks.length).toBeGreaterThan(100); for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700); }); }); describe('cleanText', () => { it('joins words hyphenated across a line break', () => { expect(cleanText('the medica-\ntion was given')).toBe('the medication was given'); }); it('keeps real compounds across a line break', () => { expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan'); }); it('collapses 3+ newlines to a paragraph break', () => { expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb'); expect(cleanText('a\n\nb')).toBe('a\n\nb'); }); it('strips control characters but keeps tabs and newlines', () => { expect(cleanText('he\u0000l\u0007lo​ ­world\ttab\nnext')).toBe('hello world\ttab\nnext'); }); it('normalises CRLF and trims', () => { expect(cleanText(' a\r\nb \r\n')).toBe('a\nb'); expect(cleanText('')).toBe(''); }); it('is idempotent', () => { const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny  z'); expect(cleanText(once)).toBe(once); }); });