/** * Text normalisation, token estimation and retrieval chunking. * Pure functions, no dependencies. Everything here is linear-time on its input. */ import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts'; // --------------------------------------------------------------------------- // Token estimation // --------------------------------------------------------------------------- /** * Un-rounded token estimate. Sub-additive under concatenation * (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and * still guarantee the final `estimateTokens(chunk)` stays under the ceiling. */ function rawTokens(text: string): number { let total = 0; let prevWhitespace = false; for (let i = 0; i < text.length; i++) { const c = text.charCodeAt(i); if (c === 32 || (c >= 9 && c <= 13)) { // A run of whitespace costs the same as one character of prose. if (!prevWhitespace) total += 0.25; prevWhitespace = true; continue; } prevWhitespace = false; if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25; else if (c >= 48 && c <= 57) total += 0.4; else if (c < 128) total += 0.5; else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each else total += 0.5; // accented Latin, Greek, symbols } return total; } /** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */ export function estimateTokens(text: string): number { if (!text) return 0; return Math.ceil(rawTokens(text)); } // --------------------------------------------------------------------------- // cleanText // --------------------------------------------------------------------------- const LIGATURES: Record = { 'ff': 'ff', 'fi': 'fi', 'fl': 'fl', 'ffi': 'ffi', 'ffl': 'ffl', 'ſt': 'st', 'st': 'st', }; /** First halves that normally form a real hyphenated compound ("well-known"). */ const HYPHEN_KEEP_LEFT = new Set([ 'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first', 'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part', 'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid', 'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed', ]); /** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */ const HYPHEN_KEEP_RIGHT = new Set([ 'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced', 'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk', 'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive', 'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset', ]); function isLower(code: number): boolean { return code >= 97 && code <= 122; } function isLetter(code: number): boolean { return (code >= 97 && code <= 122) || (code >= 65 && code <= 90); } /** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */ function joinHyphenatedLineBreaks(text: string): string { if (!text.includes('-\n')) return text; return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => { // Left fragment: lowercase letters, optionally led by one capital, at most 30 long. let start = offset; while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--; if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--; const left = whole.slice(start, offset); if (left.length < 2) return match; if (start > 0) { const before = whole.charCodeAt(start - 1); // Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it. if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match; } const rightStart = offset + match.length; let rightEnd = rightStart; while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++; const right = whole.slice(rightStart, rightEnd); if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z" if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match; return ''; }); } /** * Normalises extracted text: newlines, unicode spaces, control characters, * ligatures, trailing spaces, runs of blank lines and words hyphenated across * a line break. Idempotent. */ export function cleanText(text: string): string { if (!text) return ''; let t = text; if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n'); t = t.replace(/[\u2028\u2029\f\v]/g, '\n'); t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' '); // NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM. t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, ''); t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch); // Trailing whitespace per line (done by splitting so a huge run of spaces stays linear). t = t .split('\n') .map((line) => line.trimEnd()) .join('\n'); t = joinHyphenatedLineBreaks(t); t = t.replace(/\n{3,}/g, '\n\n'); return t.trim(); } // --------------------------------------------------------------------------- // Repeated page headers / footers // --------------------------------------------------------------------------- function edgeLineKey(line: string): string | null { const trimmed = line.trim(); if (!trimmed || trimmed.length > 100) return null; return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' '); } /** * Removes running headers/footers and page numbers from per-page text, but only * when the same first or last line (digits ignored) shows up on at least 60% of * the pages that have text, and there are at least four such pages. Chapter * headings that change through a book are therefore left alone. */ export function stripRepeatedPageLines(pages: string[]): string[] { let current = pages.map((page) => page.split('\n')); for (let pass = 0; pass < 2; pass++) { const firstIdx: number[] = []; const lastIdx: number[] = []; const topCounts = new Map(); const bottomCounts = new Map(); let textPages = 0; for (let p = 0; p < current.length; p++) { const lines = current[p]; let first = 0; while (first < lines.length && !lines[first].trim()) first++; let last = lines.length - 1; while (last >= 0 && !lines[last].trim()) last--; firstIdx.push(first < lines.length ? first : -1); lastIdx.push(last); if (last < 0) continue; textPages++; const top = edgeLineKey(lines[first]); if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1); if (last !== first) { const bottom = edgeLineKey(lines[last]); if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1); } } if (textPages < 4) break; const threshold = Math.max(3, Math.ceil(textPages * 0.6)); let removed = false; current = current.map((lines, p) => { const first = firstIdx[p]; const last = lastIdx[p]; if (last < 0) return lines; let from = 0; let to = lines.length; const top = edgeLineKey(lines[first]); if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1; if (last !== first) { const bottom = edgeLineKey(lines[last]); if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last; } if (from === 0 && to === lines.length) return lines; removed = true; return lines.slice(from, to); }); if (!removed) break; } return current.map((lines) => lines.join('\n')); } // --------------------------------------------------------------------------- // Chunking // --------------------------------------------------------------------------- /** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */ interface Atom { text: string; raw: number; sep: string; sepRaw: number; /** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */ strength: number; } const WHITESPACE_RAW = 0.25; /** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */ const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/; const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/; const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/; const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/; function sepRawOf(sep: string): number { return sep ? WHITESPACE_RAW : 0; } /** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */ function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void { let start = 0; let acc = 0; let first = true; const push = (end: number) => { const text = word.slice(start, end); out.push({ text, raw: rawTokens(text), sep: first ? sep : '', sepRaw: first ? sepRawOf(sep) : 0, strength: first ? strength : -1, }); first = false; start = end; acc = 0; }; for (let i = 0; i < word.length; i++) { const c = word.charCodeAt(i); const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff; const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost if (acc + weight > limit && i > start && !isLowSurrogate) push(i); acc += weight; } if (start < word.length) push(word.length); } function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void { const raw = rawTokens(sentence); if (raw <= atomMax) { out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength }); return; } // Oversized sentence (or table row, or blob): fall back to groups of words. const groupMax = Math.max(1, Math.min(atomMax, 24)); const words = sentence.split(/[ \t]+/); // Keep leading indentation attached to the first word. const indent = /^[ \t]+/.exec(sentence)?.[0] ?? ''; let group = ''; let groupRaw = 0; let first = true; const flush = () => { if (!group) return; out.push({ text: group, raw: rawTokens(group), sep: first ? sep : ' ', sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW, strength: first ? strength : 0, }); first = false; group = ''; groupRaw = 0; }; let pendingIndent = indent; for (const word of words) { if (!word) continue; const piece = pendingIndent + word; pendingIndent = ''; const wordRaw = rawTokens(piece); if (wordRaw > atomMax) { flush(); hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0); first = false; continue; } if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush(); group = group ? `${group} ${piece}` : piece; groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0); } flush(); } /** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */ function atomize(text: string, atomMax: number): Atom[] { const atoms: Atom[] = []; let prevLine = ''; for (const para of text.split(/\n{2,}/)) { let firstInPara = true; for (const rawLine of para.split('\n')) { const line = rawLine.trimEnd(); if (!line.trim()) continue; let sep = ''; let strength = 0; if (atoms.length > 0) { const afterHeading = HEADING_LINE.test(prevLine); const isHeading = HEADING_LINE.test(line); if (firstInPara) { sep = '\n\n'; strength = isHeading ? 4 : afterHeading ? -0.5 : 3; } else { sep = '\n'; if (isHeading) strength = 4; else if (afterHeading) strength = -0.5; else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2; else strength = 0.5; } } firstInPara = false; prevLine = line; if (rawTokens(line) <= Math.min(atomMax, 12)) { // Short line: not worth looking for sentences. pushSentence(line, atomMax, atoms, sep, strength); continue; } let firstSentence = true; for (const sentence of line.split(SENTENCE_BOUNDARY)) { if (!sentence) continue; pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1); firstSentence = false; } } } return atoms; } function joinAtoms(atoms: Atom[], from: number, to: number): string { let text = ''; for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text; return text; } /** Trailing whole words of `text` worth at most `budget` tokens. */ function tailWords(text: string, budget: number): string { let end = text.length; let cut = text.length; let acc = 0; while (end > 0) { let wordEnd = end; while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--; if (wordEnd === 0) break; const space = text.lastIndexOf(' ', wordEnd - 1); const word = text.slice(space + 1, wordEnd); const cost = rawTokens(word) + WHITESPACE_RAW; if (acc + cost > budget) break; acc += cost; cut = space + 1; end = space < 0 ? 0 : space; } return text.slice(cut).trim(); } /** * Splits one oversized section into balanced pieces of at most `bodyBudget` * tokens, cutting at the strongest boundary near the balanced size, and * prefixing each later piece with up to `overlap` tokens of trailing context. */ function splitSection(text: string, bodyBudget: number, overlap: number): string[] { const atomMax = Math.max(1, bodyBudget / 4); const atoms = atomize(text, atomMax); const n = atoms.length; const pieces: string[] = []; if (n === 0) return pieces; // suffix[i] = tokens of atoms i..n-1 including the separators before each of them. const suffix = new Float64Array(n + 1); for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw; let start = 0; let prevStart = -1; while (start < n) { const remaining = suffix[start] - atoms[start].sepRaw; let end: number; if (remaining <= bodyBudget) { end = n; } else { const goal = remaining / Math.ceil(remaining / bodyBudget); const minFill = goal * 0.6; let acc = 0; let best = -1; let bestStrength = -Infinity; let bestDistance = Infinity; let i = start; for (; i < n; i++) { const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0); if (i > start && acc + add > bodyBudget) break; acc += add; if (i + 1 < n && acc >= minFill) { const strength = atoms[i + 1].strength; const distance = Math.abs(acc - goal); if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) { best = i; bestStrength = strength; bestDistance = distance; } } } end = best >= 0 ? best + 1 : Math.max(start + 1, i); } let prefix = ''; if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) { // Trailing context from the previous piece: whole atoms if they fit, else trailing words. let from = start; let acc = 0; while (from - 1 > prevStart) { const candidate = atoms[from - 1]; const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0); if (acc + cost > overlap) break; acc += cost; from--; if (candidate.strength >= 4) break; // do not reach back past a heading } if (from < start) prefix = joinAtoms(atoms, from, start); else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap); } const body = joinAtoms(atoms, start, end); pieces.push(prefix ? prefix + atoms[start].sep + body : body); prevStart = start; start = end; } return pieces; } function positiveInt(value: number | undefined, fallback: number, min: number): number { if (typeof value !== 'number' || !Number.isFinite(value)) return fallback; return Math.max(min, Math.floor(value)); } interface Pending { text: string; heading: string; page: number | null; } /** * Builds retrieval chunks from the sections of ONE document. * * - Consecutive small sections (short slides, short pages) are packed together * up to `targetTokens`; the chunk keeps the first section's heading and the * full page range. * - A section that fits comfortably under the ceiling stays whole so chunks * line up with pages/slides. * - Larger sections are split into balanced pieces at paragraph, then * sentence, then word boundaries, with `overlapTokens` of repeated context * between consecutive pieces of the same section only. */ export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] { const target = positiveInt(opts.targetTokens, 450, 8); const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8)); const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2)); /** Sections up to this size are kept in one piece. */ const wholeLimit = Math.floor(target + (max - target) / 2); /** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */ const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1)); const chunks: DocChunk[] = []; const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => { const trimmed = text.trim(); if (!trimmed) return; chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) }); }; let pending: Pending[] = []; let pendingRaw = 0; const flush = () => { if (pending.length === 0) return; let pageStart: number | null = null; let pageEnd: number | null = null; for (const item of pending) { if (item.page === null) continue; pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page); pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page); } emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd); pending = []; pendingRaw = 0; }; let lastHeading = ''; for (const section of sections ?? []) { if (!section || typeof section.text !== 'string') continue; const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : ''; if (ownHeading) lastHeading = ownHeading; const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim(); if (!text) continue; const heading = lastHeading; const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null; const raw = rawTokens(text); if (raw > wholeLimit) { flush(); for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page); continue; } if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush(); pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0); pending.push({ text, heading, page }); } flush(); return chunks; }