Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
420
server/documents/documents.test.ts
Normal file
420
server/documents/documents.test.ts
Normal file
|
|
@ -0,0 +1,420 @@
|
|||
import { strToU8, zipSync } from 'fflate';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { chunkSections, cleanText, estimateTokens } from './chunk.ts';
|
||||
import { detectKind, extractDocument } from './extract.ts';
|
||||
import { buildDocx, buildLyingZip, buildPdf, buildPptx, TINY_GIF, TINY_JPEG, TINY_PNG, TINY_WEBP } from './testFixtures.ts';
|
||||
import { ExtractionError, type ExtractedSection } from './types.ts';
|
||||
|
||||
const enc = (s: string) => new TextEncoder().encode(s);
|
||||
const allText = (r: { sections: ExtractedSection[] }) => r.sections.map((s) => s.text).join('\n');
|
||||
const OLE = Uint8Array.from([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1, 0, 0, 0, 0]);
|
||||
/** A UTF-8 round trip turns lone surrogates into U+FFFD, so equality means well-formed. */
|
||||
const wellFormed = (s: string) => Buffer.from(s, 'utf8').toString('utf8') === s;
|
||||
|
||||
async function codeOf(promise: Promise<unknown>): Promise<string> {
|
||||
try {
|
||||
await promise;
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ExtractionError);
|
||||
return (e as ExtractionError).code;
|
||||
}
|
||||
throw new Error('expected rejection');
|
||||
}
|
||||
|
||||
describe('detectKind', () => {
|
||||
const pdf = buildPdf([{ lines: ['hello'] }]);
|
||||
const docx = buildDocx([{ type: 'p', text: 'hi' }]);
|
||||
const pptx = buildPptx([{ file: 1, title: 'T' }]);
|
||||
|
||||
it('detects by magic bytes', () => {
|
||||
expect(detectKind('a.pdf', undefined, pdf)).toBe('pdf');
|
||||
expect(detectKind('a.png', undefined, TINY_PNG)).toBe('image');
|
||||
expect(detectKind('a.jpg', undefined, TINY_JPEG)).toBe('image');
|
||||
expect(detectKind('a.gif', undefined, TINY_GIF)).toBe('image');
|
||||
expect(detectKind('a.webp', undefined, TINY_WEBP)).toBe('image');
|
||||
expect(detectKind('a.docx', undefined, docx)).toBe('docx');
|
||||
expect(detectKind('a.pptx', undefined, pptx)).toBe('pptx');
|
||||
});
|
||||
|
||||
it('trusts magic bytes over a wrong extension', () => {
|
||||
expect(detectKind('notes.txt', undefined, pdf)).toBe('pdf');
|
||||
expect(detectKind('deck.docx', undefined, pptx)).toBe('pptx');
|
||||
expect(detectKind('file.pdf', undefined, TINY_PNG)).toBe('image');
|
||||
expect(detectKind('thing.bin', undefined, docx)).toBe('docx');
|
||||
});
|
||||
|
||||
it('uses the extension for text and markdown', () => {
|
||||
expect(detectKind('a.txt', undefined, enc('hello'))).toBe('text');
|
||||
expect(detectKind('a.csv', undefined, enc('a,b\n1,2'))).toBe('text');
|
||||
expect(detectKind('a.md', undefined, enc('# hi'))).toBe('markdown');
|
||||
expect(detectKind('a.markdown', undefined, enc('# hi'))).toBe('markdown');
|
||||
expect(detectKind('README', 'text/markdown', enc('# hi'))).toBe('markdown');
|
||||
});
|
||||
|
||||
it('rejects unknown binary, legacy .doc, plain zips and empty input', () => {
|
||||
expect(detectKind('a.bin', undefined, Uint8Array.from([1, 2, 3, 4, 5]))).toBeNull();
|
||||
expect(detectKind('a.txt', undefined, Uint8Array.from([0, 1, 2, 3, 0, 0]))).toBeNull();
|
||||
expect(detectKind('old.doc', undefined, OLE)).toBeNull();
|
||||
expect(detectKind('a.txt', undefined, new Uint8Array(0))).toBeNull();
|
||||
expect(detectKind('a.zip', undefined, zipSync({ 'x.txt': strToU8('hi') }))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: PDF', () => {
|
||||
it('extracts multi-page text with page numbers, count and title', async () => {
|
||||
const bytes = buildPdf(
|
||||
[{ lines: ['Heart failure overview', 'Left sided failure causes congestion.'] }, { lines: ['Digoxin toxicity causes nausea.'] }],
|
||||
{ title: 'Cardiology Notes' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
||||
expect(r.kind).toBe('pdf');
|
||||
expect(r.pageCount).toBe(2);
|
||||
expect(r.title).toBe('Cardiology Notes');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
expect(r.warnings).toEqual([]);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2]);
|
||||
expect(r.sections[0].text).toContain('Heart failure overview');
|
||||
expect(r.sections[1].text).toContain('Digoxin');
|
||||
});
|
||||
|
||||
it('warns about a page without text but keeps page numbers', async () => {
|
||||
const long = 'Readable text on this page, long enough to avoid OCR. ';
|
||||
const bytes = buildPdf([{ lines: [long.repeat(2)] }, { lines: [] }, { lines: [long.repeat(2)] }]);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pdf' });
|
||||
expect(r.pageCount).toBe(3);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 3]);
|
||||
expect(r.warnings.some((w) => /1 of 3 pages had no extractable text/.test(w))).toBe(true);
|
||||
});
|
||||
|
||||
it('flags an all-empty scanned PDF as needing OCR', async () => {
|
||||
const r = await extractDocument({ bytes: buildPdf([{ lines: [] }, { lines: [] }]), filename: 'scan.pdf' });
|
||||
expect(r.needsOcr).toBe(true);
|
||||
expect(r.sections).toEqual([]);
|
||||
expect(r.pageCount).toBe(2);
|
||||
expect(r.warnings.join(' ')).toMatch(/scanned/);
|
||||
});
|
||||
|
||||
it('reports progress per page', async () => {
|
||||
const seen: Array<[number, number]> = [];
|
||||
await extractDocument({ bytes: buildPdf([{ lines: ['a'] }, { lines: ['b'] }]), filename: 'a.pdf', onProgress: (d, t) => seen.push([d, t]) });
|
||||
expect(seen).toEqual([
|
||||
[1, 2],
|
||||
[2, 2],
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: DOCX', () => {
|
||||
it('turns headings into sections and keeps lists and tables readable', async () => {
|
||||
const bytes = buildDocx(
|
||||
[
|
||||
{ type: 'heading', level: 1, text: 'Cardiac Drugs' },
|
||||
{ type: 'p', text: 'Intro paragraph about drugs.' },
|
||||
{ type: 'bullet', text: 'Beta blockers' },
|
||||
{ type: 'bullet', text: 'ACE inhibitors' },
|
||||
{ type: 'heading', level: 2, text: 'Dosing' },
|
||||
{ type: 'number', text: 'Check heart rate' },
|
||||
{ type: 'number', text: 'Give the dose' },
|
||||
{
|
||||
type: 'table',
|
||||
rows: [
|
||||
['Drug', 'Dose'],
|
||||
['Digoxin', '0.25 mg'],
|
||||
],
|
||||
},
|
||||
],
|
||||
{ title: 'Drug Guide' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.docx' });
|
||||
expect(r.kind).toBe('docx');
|
||||
expect(r.title).toBe('Drug Guide');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Cardiac Drugs', 'Dosing']);
|
||||
const first = r.sections[0].text;
|
||||
expect(first).toContain('Intro paragraph');
|
||||
expect(first).toContain('Beta blockers');
|
||||
expect(first).toContain('ACE inhibitors');
|
||||
const second = r.sections[1].text;
|
||||
expect(second).toContain('Check heart rate');
|
||||
expect(second).toContain('Digoxin');
|
||||
expect(second).toContain('0.25 mg');
|
||||
});
|
||||
|
||||
it('rejects a docx with no text as empty', async () => {
|
||||
const bytes = buildDocx([{ type: 'p', text: '' }]);
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('empty');
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: PPTX', () => {
|
||||
it('follows presentation order, numbers slides, and reads titles, bullets, notes', async () => {
|
||||
const bytes = buildPptx(
|
||||
[
|
||||
{ file: 3, title: 'Opening', bullets: ['Welcome', { text: 'Sub point', level: 1 }], notes: ['Say hello slowly'] },
|
||||
{ file: 1, title: 'Middle', textBox: ['Free text box content here'] },
|
||||
{ file: 2 },
|
||||
{ file: 4, title: 'Closing', bullets: ['Thanks'] },
|
||||
],
|
||||
{ title: 'Deck Title' },
|
||||
);
|
||||
const r = await extractDocument({ bytes, filename: 'a.pptx' });
|
||||
expect(r.kind).toBe('pptx');
|
||||
expect(r.title).toBe('Deck Title');
|
||||
expect(r.pageCount).toBe(4);
|
||||
expect(r.sections.map((s) => s.page)).toEqual([1, 2, 4]);
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Opening', 'Middle', 'Closing']);
|
||||
expect(r.sections[0].text).toContain('Welcome');
|
||||
expect(r.sections[0].text).toContain('Sub point');
|
||||
expect(r.sections[0].text).toContain('Say hello slowly');
|
||||
expect(r.sections[1].text).toContain('Free text box content here');
|
||||
expect(r.sections[0].text).not.toMatch(/\b9\b/); // slide-number placeholder ignored
|
||||
expect(r.warnings).toEqual([]); // one empty slide is not worth a warning
|
||||
});
|
||||
|
||||
it('reads tables', async () => {
|
||||
const table = [
|
||||
['A', 'B'],
|
||||
['x', 'y'],
|
||||
];
|
||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1, title: 'T', table }]), filename: 'a.pptx' });
|
||||
expect(allText(r)).toContain('x');
|
||||
expect(allText(r)).toContain('B');
|
||||
});
|
||||
|
||||
it('warns when most slides have no text', async () => {
|
||||
const r = await extractDocument({ bytes: buildPptx([{ file: 1 }, { file: 2 }, { file: 3, title: 'Only one' }]), filename: 'a.pptx' });
|
||||
expect(r.warnings.some((w) => /2 of 3 slides had no text/.test(w))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: text, markdown, image', () => {
|
||||
it('splits markdown by heading and ignores headings inside code fences', async () => {
|
||||
const md = '# Title\n\nIntro text.\n\n## Part One\n\nOne body.\n\n```\n# not a heading\n```\n\n## Part Two\n\nTwo body.';
|
||||
const r = await extractDocument({ bytes: enc(md), filename: 'a.md' });
|
||||
expect(r.kind).toBe('markdown');
|
||||
expect(r.title).toBe('Title');
|
||||
expect(r.sections.map((s) => s.heading)).toEqual(['Title', 'Part One', 'Part Two']);
|
||||
expect(r.sections[1].text).toContain('# not a heading');
|
||||
expect(r.needsOcr).toBe(false);
|
||||
});
|
||||
|
||||
it('extracts plain text and falls back for non-UTF-8 with a warning', async () => {
|
||||
const r = await extractDocument({ bytes: enc('line one\n\nline two'), filename: 'a.txt' });
|
||||
expect(r.kind).toBe('text');
|
||||
expect(allText(r)).toContain('line two');
|
||||
const latin = Uint8Array.from([0x63, 0x61, 0x66, 0xe9]); // windows-1252 "cafe" with e-acute
|
||||
const r2 = await extractDocument({ bytes: latin, filename: 'a.txt' });
|
||||
expect(allText(r2)).toBe('café');
|
||||
expect(r2.warnings.length).toBe(1);
|
||||
});
|
||||
|
||||
it('groups long plain text into several bounded sections', async () => {
|
||||
const text = Array.from({ length: 400 }, (_, i) => `Paragraph ${i} with some words in it.`).join('\n\n');
|
||||
const r = await extractDocument({ bytes: enc(text), filename: 'a.txt' });
|
||||
expect(r.sections.length).toBeGreaterThan(1);
|
||||
for (const s of r.sections) expect(s.text.length).toBeLessThan(7000);
|
||||
});
|
||||
|
||||
it('returns images with needsOcr and no sections', async () => {
|
||||
const r = await extractDocument({ bytes: TINY_PNG, filename: 'x.png' });
|
||||
expect(r).toMatchObject({ kind: 'image', sections: [], needsOcr: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractDocument: errors', () => {
|
||||
it('rejects an empty file', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: new Uint8Array(0), filename: 'a.pdf' }))).toBe('empty');
|
||||
});
|
||||
|
||||
it('rejects whitespace-only text as empty', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: enc(' \n\n '), filename: 'a.txt' }))).toBe('empty');
|
||||
});
|
||||
|
||||
it('rejects unsupported types and legacy .doc', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: Uint8Array.from([1, 2, 3, 4, 5, 6]), filename: 'a.bin' }))).toBe('unsupported');
|
||||
await expect(extractDocument({ bytes: OLE, filename: 'old.doc' })).rejects.toThrow(/\.doc/);
|
||||
expect(await codeOf(extractDocument({ bytes: OLE, filename: 'old.doc' }))).toBe('unsupported');
|
||||
});
|
||||
|
||||
it('rejects a password-protected OOXML wrapper as encrypted', async () => {
|
||||
const marker = Buffer.from('EncryptedPackage', 'utf16le');
|
||||
const bytes = Uint8Array.from([...OLE, ...marker]);
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.docx' }))).toBe('encrypted');
|
||||
});
|
||||
|
||||
it('rejects corrupt PDFs', async () => {
|
||||
expect(await codeOf(extractDocument({ bytes: enc('%PDF-1.4\nthis is not really a pdf'), filename: 'a.pdf' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects an encrypted PDF', async () => {
|
||||
const bytes = buildPdf([{ lines: ['secret'] }], { encrypted: true });
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pdf' }))).toBe('encrypted');
|
||||
});
|
||||
|
||||
it('rejects a corrupt zip posing as docx/pptx', async () => {
|
||||
const good = buildDocx([{ type: 'p', text: 'hello world' }]);
|
||||
const broken = good.slice(0, Math.floor(good.length / 2));
|
||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.docx' }))).toBe('corrupt');
|
||||
expect(await codeOf(extractDocument({ bytes: broken, filename: 'a.pptx' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects a pptx zip that has no readable slides', async () => {
|
||||
const bytes = zipSync({ 'ppt/presentation.xml': strToU8('<p:presentation/>') });
|
||||
expect(await codeOf(extractDocument({ bytes, filename: 'a.pptx' }))).toBe('corrupt');
|
||||
});
|
||||
|
||||
it('rejects a zip bomb whose headers lie about entry size', async () => {
|
||||
const bytes = buildLyingZip('word/media/bomb.bin', 80 * 1024 * 1024);
|
||||
const code = await codeOf(extractDocument({ bytes, filename: 'bomb.docx' }));
|
||||
expect(['too_large', 'corrupt']).toContain(code);
|
||||
}, 30000);
|
||||
|
||||
it('honours an already-aborted signal', async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
expect(await codeOf(extractDocument({ bytes: enc('hi'), filename: 'a.txt', signal: ac.signal }))).toBe('failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('chunkSections', () => {
|
||||
const words = (n: number, prefix = 'word') => Array.from({ length: n }, (_, i) => `${prefix}${i}`).join(' ');
|
||||
const opts = { targetTokens: 100, maxTokens: 150, overlapTokens: 20 };
|
||||
|
||||
it('keeps every chunk within maxTokens with contiguous 0-based seq', () => {
|
||||
const sections: ExtractedSection[] = Array.from({ length: 12 }, (_, i) => ({
|
||||
text: Array.from({ length: 30 }, (_, j) => `Sentence number ${j} of section ${i} talks about things.`).join(' '),
|
||||
page: i + 1,
|
||||
}));
|
||||
const chunks = chunkSections(sections, opts);
|
||||
expect(chunks.length).toBeGreaterThan(12);
|
||||
chunks.forEach((c, i) => {
|
||||
expect(c.seq).toBe(i);
|
||||
expect(c.tokens).toBe(estimateTokens(c.text));
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
});
|
||||
});
|
||||
|
||||
it('packs small sections together with correct page range and first heading', () => {
|
||||
const sections: ExtractedSection[] = [
|
||||
{ text: 'Alpha text.', page: 1, heading: 'First' },
|
||||
{ text: 'Beta text.', page: 2 },
|
||||
{ text: 'Gamma text.', page: 3, heading: 'Third' },
|
||||
];
|
||||
const chunks = chunkSections(sections, opts);
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0]).toMatchObject({ pageStart: 1, pageEnd: 3, heading: 'First', seq: 0 });
|
||||
expect(chunks[0].text).toContain('Gamma');
|
||||
});
|
||||
|
||||
it('starts a new chunk when packing would exceed the target and tracks ranges', () => {
|
||||
const big = (i: number) => ({ text: words(60, `s${i}x`), page: i });
|
||||
const chunks = chunkSections([big(1), big(2), big(3), big(4)], { targetTokens: 100, maxTokens: 200, overlapTokens: 0 });
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) {
|
||||
expect(c.pageStart).not.toBeNull();
|
||||
expect(c.pageEnd!).toBeGreaterThanOrEqual(c.pageStart!);
|
||||
}
|
||||
expect(chunks[0].pageStart).toBe(1);
|
||||
expect(chunks[chunks.length - 1].pageEnd).toBe(4);
|
||||
});
|
||||
|
||||
it('uses null pages and empty heading when sections have none', () => {
|
||||
const [c] = chunkSections([{ text: 'No pages here.' }]);
|
||||
expect(c.pageStart).toBeNull();
|
||||
expect(c.pageEnd).toBeNull();
|
||||
expect(c.heading).toBe('');
|
||||
});
|
||||
|
||||
it('splits oversized sections without cutting words', () => {
|
||||
const text = words(800, 'term');
|
||||
const chunks = chunkSections([{ text, page: 5, heading: 'Big' }], opts);
|
||||
expect(chunks.length).toBeGreaterThan(3);
|
||||
const vocabulary = new Set(text.split(' '));
|
||||
for (const c of chunks) {
|
||||
expect(c.pageStart).toBe(5);
|
||||
expect(c.pageEnd).toBe(5);
|
||||
expect(c.heading).toBe('Big');
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
for (const w of c.text.split(/\s+/)) expect(vocabulary.has(w)).toBe(true);
|
||||
}
|
||||
const seen = new Set(chunks.flatMap((c) => c.text.split(/\s+/)));
|
||||
expect(seen.size).toBe(800);
|
||||
});
|
||||
|
||||
it('survives a giant unbroken string within the limit', () => {
|
||||
const blob = 'x'.repeat(200_000);
|
||||
const t0 = Date.now();
|
||||
const chunks = chunkSections([{ text: blob }], opts);
|
||||
expect(Date.now() - t0).toBeLessThan(3000);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
expect(chunks.reduce((n, c) => n + c.text.length, 0)).toBeGreaterThanOrEqual(200_000);
|
||||
});
|
||||
|
||||
it('does not split surrogate pairs in unbroken emoji runs', () => {
|
||||
const chunks = chunkSections([{ text: '\u{1F600}'.repeat(5000) }], opts);
|
||||
expect(chunks.length).toBeGreaterThan(1);
|
||||
for (const c of chunks) {
|
||||
expect(wellFormed(c.text)).toBe(true);
|
||||
expect(c.tokens).toBeLessThanOrEqual(150);
|
||||
}
|
||||
});
|
||||
|
||||
it('is deterministic', () => {
|
||||
const sections = [
|
||||
{ text: words(900), page: 1, heading: 'H' },
|
||||
{ text: 'Short one.', page: 2 },
|
||||
];
|
||||
expect(chunkSections(sections, opts)).toEqual(chunkSections(sections, opts));
|
||||
});
|
||||
|
||||
it('drops empty and whitespace-only sections', () => {
|
||||
expect(chunkSections([{ text: '' }, { text: ' \n\t ' }])).toEqual([]);
|
||||
const chunks = chunkSections([
|
||||
{ text: ' ', page: 1 },
|
||||
{ text: 'Real content.', page: 2 },
|
||||
{ text: '\n', page: 3 },
|
||||
]);
|
||||
expect(chunks).toHaveLength(1);
|
||||
expect(chunks[0]).toMatchObject({ pageStart: 2, pageEnd: 2 });
|
||||
});
|
||||
|
||||
it('chunks about 2 MB of text quickly', () => {
|
||||
const para = 'The quick brown fox jumps over the lazy dog. Pharmacology covers drug action and dosing.\n\n';
|
||||
const text = para.repeat(Math.ceil((2 * 1024 * 1024) / para.length));
|
||||
const t0 = Date.now();
|
||||
const chunks = chunkSections([{ text }]);
|
||||
expect(Date.now() - t0).toBeLessThan(3000);
|
||||
expect(chunks.length).toBeGreaterThan(100);
|
||||
for (const c of chunks) expect(c.tokens).toBeLessThanOrEqual(700);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanText', () => {
|
||||
it('joins words hyphenated across a line break', () => {
|
||||
expect(cleanText('the medica-\ntion was given')).toBe('the medication was given');
|
||||
});
|
||||
|
||||
it('keeps real compounds across a line break', () => {
|
||||
expect(cleanText('a long-\nterm plan')).toBe('a long-\nterm plan');
|
||||
});
|
||||
|
||||
it('collapses 3+ newlines to a paragraph break', () => {
|
||||
expect(cleanText('a\n\n\n\n\nb')).toBe('a\n\nb');
|
||||
expect(cleanText('a\n\nb')).toBe('a\n\nb');
|
||||
});
|
||||
|
||||
it('strips control characters but keeps tabs and newlines', () => {
|
||||
expect(cleanText('he\u0000l\u0007lo world\ttab\nnext')).toBe('hello world\ttab\nnext');
|
||||
});
|
||||
|
||||
it('normalises CRLF and trims', () => {
|
||||
expect(cleanText(' a\r\nb \r\n')).toBe('a\nb');
|
||||
expect(cleanText('')).toBe('');
|
||||
});
|
||||
|
||||
it('is idempotent', () => {
|
||||
const once = cleanText('x\u0000 medica-\ntion\n\n\n\ny z');
|
||||
expect(cleanText(once)).toBe(once);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue