Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
514
server/documents/chunk.ts
Normal file
514
server/documents/chunk.ts
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
/**
|
||||
* Text normalisation, token estimation and retrieval chunking.
|
||||
* Pure functions, no dependencies. Everything here is linear-time on its input.
|
||||
*/
|
||||
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Token estimation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Un-rounded token estimate. Sub-additive under concatenation
|
||||
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
|
||||
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
|
||||
*/
|
||||
function rawTokens(text: string): number {
|
||||
let total = 0;
|
||||
let prevWhitespace = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text.charCodeAt(i);
|
||||
if (c === 32 || (c >= 9 && c <= 13)) {
|
||||
// A run of whitespace costs the same as one character of prose.
|
||||
if (!prevWhitespace) total += 0.25;
|
||||
prevWhitespace = true;
|
||||
continue;
|
||||
}
|
||||
prevWhitespace = false;
|
||||
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
|
||||
else if (c >= 48 && c <= 57) total += 0.4;
|
||||
else if (c < 128) total += 0.5;
|
||||
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
|
||||
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
|
||||
else total += 0.5; // accented Latin, Greek, symbols
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
|
||||
export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
return Math.ceil(rawTokens(text));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// cleanText
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const LIGATURES: Record<string, string> = {
|
||||
'ff': 'ff',
|
||||
'fi': 'fi',
|
||||
'fl': 'fl',
|
||||
'ffi': 'ffi',
|
||||
'ffl': 'ffl',
|
||||
'ſt': 'st',
|
||||
'st': 'st',
|
||||
};
|
||||
|
||||
/** First halves that normally form a real hyphenated compound ("well-known"). */
|
||||
const HYPHEN_KEEP_LEFT = new Set([
|
||||
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
|
||||
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
|
||||
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
|
||||
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
|
||||
]);
|
||||
|
||||
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
|
||||
const HYPHEN_KEEP_RIGHT = new Set([
|
||||
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
|
||||
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
|
||||
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
|
||||
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
|
||||
]);
|
||||
|
||||
function isLower(code: number): boolean {
|
||||
return code >= 97 && code <= 122;
|
||||
}
|
||||
|
||||
function isLetter(code: number): boolean {
|
||||
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
|
||||
}
|
||||
|
||||
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
|
||||
function joinHyphenatedLineBreaks(text: string): string {
|
||||
if (!text.includes('-\n')) return text;
|
||||
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
|
||||
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
|
||||
let start = offset;
|
||||
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
|
||||
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
|
||||
const left = whole.slice(start, offset);
|
||||
if (left.length < 2) return match;
|
||||
if (start > 0) {
|
||||
const before = whole.charCodeAt(start - 1);
|
||||
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
|
||||
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
|
||||
}
|
||||
const rightStart = offset + match.length;
|
||||
let rightEnd = rightStart;
|
||||
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
|
||||
const right = whole.slice(rightStart, rightEnd);
|
||||
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
|
||||
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
|
||||
return '';
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalises extracted text: newlines, unicode spaces, control characters,
|
||||
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
|
||||
* a line break. Idempotent.
|
||||
*/
|
||||
export function cleanText(text: string): string {
|
||||
if (!text) return '';
|
||||
let t = text;
|
||||
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
|
||||
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
|
||||
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
|
||||
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
|
||||
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
|
||||
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
|
||||
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
|
||||
t = t
|
||||
.split('\n')
|
||||
.map((line) => line.trimEnd())
|
||||
.join('\n');
|
||||
t = joinHyphenatedLineBreaks(t);
|
||||
t = t.replace(/\n{3,}/g, '\n\n');
|
||||
return t.trim();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Repeated page headers / footers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function edgeLineKey(line: string): string | null {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.length > 100) return null;
|
||||
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes running headers/footers and page numbers from per-page text, but only
|
||||
* when the same first or last line (digits ignored) shows up on at least 60% of
|
||||
* the pages that have text, and there are at least four such pages. Chapter
|
||||
* headings that change through a book are therefore left alone.
|
||||
*/
|
||||
export function stripRepeatedPageLines(pages: string[]): string[] {
|
||||
let current = pages.map((page) => page.split('\n'));
|
||||
for (let pass = 0; pass < 2; pass++) {
|
||||
const firstIdx: number[] = [];
|
||||
const lastIdx: number[] = [];
|
||||
const topCounts = new Map<string, number>();
|
||||
const bottomCounts = new Map<string, number>();
|
||||
let textPages = 0;
|
||||
for (let p = 0; p < current.length; p++) {
|
||||
const lines = current[p];
|
||||
let first = 0;
|
||||
while (first < lines.length && !lines[first].trim()) first++;
|
||||
let last = lines.length - 1;
|
||||
while (last >= 0 && !lines[last].trim()) last--;
|
||||
firstIdx.push(first < lines.length ? first : -1);
|
||||
lastIdx.push(last);
|
||||
if (last < 0) continue;
|
||||
textPages++;
|
||||
const top = edgeLineKey(lines[first]);
|
||||
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
|
||||
if (last !== first) {
|
||||
const bottom = edgeLineKey(lines[last]);
|
||||
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
if (textPages < 4) break;
|
||||
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
|
||||
let removed = false;
|
||||
current = current.map((lines, p) => {
|
||||
const first = firstIdx[p];
|
||||
const last = lastIdx[p];
|
||||
if (last < 0) return lines;
|
||||
let from = 0;
|
||||
let to = lines.length;
|
||||
const top = edgeLineKey(lines[first]);
|
||||
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
|
||||
if (last !== first) {
|
||||
const bottom = edgeLineKey(lines[last]);
|
||||
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
|
||||
}
|
||||
if (from === 0 && to === lines.length) return lines;
|
||||
removed = true;
|
||||
return lines.slice(from, to);
|
||||
});
|
||||
if (!removed) break;
|
||||
}
|
||||
return current.map((lines) => lines.join('\n'));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chunking
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
|
||||
interface Atom {
|
||||
text: string;
|
||||
raw: number;
|
||||
sep: string;
|
||||
sepRaw: number;
|
||||
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
|
||||
strength: number;
|
||||
}
|
||||
|
||||
const WHITESPACE_RAW = 0.25;
|
||||
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
|
||||
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
|
||||
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
|
||||
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
|
||||
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
|
||||
|
||||
function sepRawOf(sep: string): number {
|
||||
return sep ? WHITESPACE_RAW : 0;
|
||||
}
|
||||
|
||||
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
|
||||
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
|
||||
let start = 0;
|
||||
let acc = 0;
|
||||
let first = true;
|
||||
const push = (end: number) => {
|
||||
const text = word.slice(start, end);
|
||||
out.push({
|
||||
text,
|
||||
raw: rawTokens(text),
|
||||
sep: first ? sep : '',
|
||||
sepRaw: first ? sepRawOf(sep) : 0,
|
||||
strength: first ? strength : -1,
|
||||
});
|
||||
first = false;
|
||||
start = end;
|
||||
acc = 0;
|
||||
};
|
||||
for (let i = 0; i < word.length; i++) {
|
||||
const c = word.charCodeAt(i);
|
||||
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
|
||||
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
|
||||
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
|
||||
acc += weight;
|
||||
}
|
||||
if (start < word.length) push(word.length);
|
||||
}
|
||||
|
||||
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
|
||||
const raw = rawTokens(sentence);
|
||||
if (raw <= atomMax) {
|
||||
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
|
||||
return;
|
||||
}
|
||||
// Oversized sentence (or table row, or blob): fall back to groups of words.
|
||||
const groupMax = Math.max(1, Math.min(atomMax, 24));
|
||||
const words = sentence.split(/[ \t]+/);
|
||||
// Keep leading indentation attached to the first word.
|
||||
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
|
||||
let group = '';
|
||||
let groupRaw = 0;
|
||||
let first = true;
|
||||
const flush = () => {
|
||||
if (!group) return;
|
||||
out.push({
|
||||
text: group,
|
||||
raw: rawTokens(group),
|
||||
sep: first ? sep : ' ',
|
||||
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
|
||||
strength: first ? strength : 0,
|
||||
});
|
||||
first = false;
|
||||
group = '';
|
||||
groupRaw = 0;
|
||||
};
|
||||
let pendingIndent = indent;
|
||||
for (const word of words) {
|
||||
if (!word) continue;
|
||||
const piece = pendingIndent + word;
|
||||
pendingIndent = '';
|
||||
const wordRaw = rawTokens(piece);
|
||||
if (wordRaw > atomMax) {
|
||||
flush();
|
||||
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
|
||||
group = group ? `${group} ${piece}` : piece;
|
||||
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
|
||||
function atomize(text: string, atomMax: number): Atom[] {
|
||||
const atoms: Atom[] = [];
|
||||
let prevLine = '';
|
||||
for (const para of text.split(/\n{2,}/)) {
|
||||
let firstInPara = true;
|
||||
for (const rawLine of para.split('\n')) {
|
||||
const line = rawLine.trimEnd();
|
||||
if (!line.trim()) continue;
|
||||
let sep = '';
|
||||
let strength = 0;
|
||||
if (atoms.length > 0) {
|
||||
const afterHeading = HEADING_LINE.test(prevLine);
|
||||
const isHeading = HEADING_LINE.test(line);
|
||||
if (firstInPara) {
|
||||
sep = '\n\n';
|
||||
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
|
||||
} else {
|
||||
sep = '\n';
|
||||
if (isHeading) strength = 4;
|
||||
else if (afterHeading) strength = -0.5;
|
||||
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
|
||||
else strength = 0.5;
|
||||
}
|
||||
}
|
||||
firstInPara = false;
|
||||
prevLine = line;
|
||||
|
||||
if (rawTokens(line) <= Math.min(atomMax, 12)) {
|
||||
// Short line: not worth looking for sentences.
|
||||
pushSentence(line, atomMax, atoms, sep, strength);
|
||||
continue;
|
||||
}
|
||||
let firstSentence = true;
|
||||
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
|
||||
if (!sentence) continue;
|
||||
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
|
||||
firstSentence = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return atoms;
|
||||
}
|
||||
|
||||
function joinAtoms(atoms: Atom[], from: number, to: number): string {
|
||||
let text = '';
|
||||
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
|
||||
return text;
|
||||
}
|
||||
|
||||
/** Trailing whole words of `text` worth at most `budget` tokens. */
|
||||
function tailWords(text: string, budget: number): string {
|
||||
let end = text.length;
|
||||
let cut = text.length;
|
||||
let acc = 0;
|
||||
while (end > 0) {
|
||||
let wordEnd = end;
|
||||
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
|
||||
if (wordEnd === 0) break;
|
||||
const space = text.lastIndexOf(' ', wordEnd - 1);
|
||||
const word = text.slice(space + 1, wordEnd);
|
||||
const cost = rawTokens(word) + WHITESPACE_RAW;
|
||||
if (acc + cost > budget) break;
|
||||
acc += cost;
|
||||
cut = space + 1;
|
||||
end = space < 0 ? 0 : space;
|
||||
}
|
||||
return text.slice(cut).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits one oversized section into balanced pieces of at most `bodyBudget`
|
||||
* tokens, cutting at the strongest boundary near the balanced size, and
|
||||
* prefixing each later piece with up to `overlap` tokens of trailing context.
|
||||
*/
|
||||
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
|
||||
const atomMax = Math.max(1, bodyBudget / 4);
|
||||
const atoms = atomize(text, atomMax);
|
||||
const n = atoms.length;
|
||||
const pieces: string[] = [];
|
||||
if (n === 0) return pieces;
|
||||
|
||||
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
|
||||
const suffix = new Float64Array(n + 1);
|
||||
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
|
||||
|
||||
let start = 0;
|
||||
let prevStart = -1;
|
||||
while (start < n) {
|
||||
const remaining = suffix[start] - atoms[start].sepRaw;
|
||||
let end: number;
|
||||
if (remaining <= bodyBudget) {
|
||||
end = n;
|
||||
} else {
|
||||
const goal = remaining / Math.ceil(remaining / bodyBudget);
|
||||
const minFill = goal * 0.6;
|
||||
let acc = 0;
|
||||
let best = -1;
|
||||
let bestStrength = -Infinity;
|
||||
let bestDistance = Infinity;
|
||||
let i = start;
|
||||
for (; i < n; i++) {
|
||||
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
|
||||
if (i > start && acc + add > bodyBudget) break;
|
||||
acc += add;
|
||||
if (i + 1 < n && acc >= minFill) {
|
||||
const strength = atoms[i + 1].strength;
|
||||
const distance = Math.abs(acc - goal);
|
||||
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
|
||||
best = i;
|
||||
bestStrength = strength;
|
||||
bestDistance = distance;
|
||||
}
|
||||
}
|
||||
}
|
||||
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
|
||||
}
|
||||
|
||||
let prefix = '';
|
||||
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
|
||||
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
|
||||
let from = start;
|
||||
let acc = 0;
|
||||
while (from - 1 > prevStart) {
|
||||
const candidate = atoms[from - 1];
|
||||
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
|
||||
if (acc + cost > overlap) break;
|
||||
acc += cost;
|
||||
from--;
|
||||
if (candidate.strength >= 4) break; // do not reach back past a heading
|
||||
}
|
||||
if (from < start) prefix = joinAtoms(atoms, from, start);
|
||||
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
|
||||
}
|
||||
const body = joinAtoms(atoms, start, end);
|
||||
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
|
||||
prevStart = start;
|
||||
start = end;
|
||||
}
|
||||
return pieces;
|
||||
}
|
||||
|
||||
function positiveInt(value: number | undefined, fallback: number, min: number): number {
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
|
||||
return Math.max(min, Math.floor(value));
|
||||
}
|
||||
|
||||
interface Pending {
|
||||
text: string;
|
||||
heading: string;
|
||||
page: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds retrieval chunks from the sections of ONE document.
|
||||
*
|
||||
* - Consecutive small sections (short slides, short pages) are packed together
|
||||
* up to `targetTokens`; the chunk keeps the first section's heading and the
|
||||
* full page range.
|
||||
* - A section that fits comfortably under the ceiling stays whole so chunks
|
||||
* line up with pages/slides.
|
||||
* - Larger sections are split into balanced pieces at paragraph, then
|
||||
* sentence, then word boundaries, with `overlapTokens` of repeated context
|
||||
* between consecutive pieces of the same section only.
|
||||
*/
|
||||
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
|
||||
const target = positiveInt(opts.targetTokens, 450, 8);
|
||||
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
|
||||
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
|
||||
/** Sections up to this size are kept in one piece. */
|
||||
const wholeLimit = Math.floor(target + (max - target) / 2);
|
||||
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
|
||||
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
|
||||
|
||||
const chunks: DocChunk[] = [];
|
||||
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return;
|
||||
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
|
||||
};
|
||||
|
||||
let pending: Pending[] = [];
|
||||
let pendingRaw = 0;
|
||||
const flush = () => {
|
||||
if (pending.length === 0) return;
|
||||
let pageStart: number | null = null;
|
||||
let pageEnd: number | null = null;
|
||||
for (const item of pending) {
|
||||
if (item.page === null) continue;
|
||||
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
|
||||
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
|
||||
}
|
||||
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
|
||||
pending = [];
|
||||
pendingRaw = 0;
|
||||
};
|
||||
|
||||
let lastHeading = '';
|
||||
for (const section of sections ?? []) {
|
||||
if (!section || typeof section.text !== 'string') continue;
|
||||
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
|
||||
if (ownHeading) lastHeading = ownHeading;
|
||||
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
|
||||
if (!text) continue;
|
||||
const heading = lastHeading;
|
||||
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
|
||||
const raw = rawTokens(text);
|
||||
|
||||
if (raw > wholeLimit) {
|
||||
flush();
|
||||
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
|
||||
continue;
|
||||
}
|
||||
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
|
||||
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
|
||||
pending.push({ text, heading, page });
|
||||
}
|
||||
flush();
|
||||
return chunks;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue