Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

514
server/documents/chunk.ts Normal file
View file

@ -0,0 +1,514 @@
/**
* Text normalisation, token estimation and retrieval chunking.
* Pure functions, no dependencies. Everything here is linear-time on its input.
*/
import type { ChunkOptions, DocChunk, ExtractedSection } from './types.ts';
// ---------------------------------------------------------------------------
// Token estimation
// ---------------------------------------------------------------------------
/**
* Un-rounded token estimate. Sub-additive under concatenation
* (raw(a + b) <= raw(a) + raw(b)), which lets the chunker budget with sums and
* still guarantee the final `estimateTokens(chunk)` stays under the ceiling.
*/
function rawTokens(text: string): number {
let total = 0;
let prevWhitespace = false;
for (let i = 0; i < text.length; i++) {
const c = text.charCodeAt(i);
if (c === 32 || (c >= 9 && c <= 13)) {
// A run of whitespace costs the same as one character of prose.
if (!prevWhitespace) total += 0.25;
prevWhitespace = true;
continue;
}
prevWhitespace = false;
if ((c >= 97 && c <= 122) || (c >= 65 && c <= 90)) total += 0.25;
else if (c >= 48 && c <= 57) total += 0.4;
else if (c < 128) total += 0.5;
else if (c >= 0xd800 && c <= 0xdfff) total += 0.75; // half of an astral character
else if (c >= 0x2e80) total += 1; // CJK and friends: about a token each
else total += 0.5; // accented Latin, Greek, symbols
}
return total;
}
/** Fast token estimate: roughly chars/4 for English prose, heavier for digits, symbols and CJK. */
export function estimateTokens(text: string): number {
if (!text) return 0;
return Math.ceil(rawTokens(text));
}
// ---------------------------------------------------------------------------
// cleanText
// ---------------------------------------------------------------------------
const LIGATURES: Record<string, string> = {
'ff': 'ff',
'fi': 'fi',
'fl': 'fl',
'ffi': 'ffi',
'ffl': 'ffl',
'ſt': 'st',
'st': 'st',
};
/** First halves that normally form a real hyphenated compound ("well-known"). */
const HYPHEN_KEEP_LEFT = new Set([
'self', 'well', 'long', 'short', 'high', 'low', 'all', 'half', 'cross', 'non', 'ex', 'follow', 'first',
'second', 'third', 'one', 'two', 'three', 'four', 'five', 'six', 'single', 'double', 'full', 'part',
'evidence', 'patient', 'nurse', 'family', 'person', 'life', 'end', 'real', 'time', 'late', 'early', 'mid',
'so', 'quasi', 'dose', 'age', 'gram', 'broad', 'narrow', 'fast', 'slow', 'rapid', 'best', 'open', 'closed',
]);
/** Second halves that normally form a real hyphenated compound ("follow-up", "long-term"). */
const HYPHEN_KEEP_RIGHT = new Set([
'up', 'in', 'on', 'off', 'out', 'by', 'to', 'term', 'based', 'related', 'like', 'free', 'induced',
'specific', 'dependent', 'acting', 'year', 'old', 'day', 'month', 'week', 'hour', 'minute', 'line', 'risk',
'dose', 'level', 'care', 'centered', 'centred', 'led', 'associated', 'mediated', 'resistant', 'sensitive',
'negative', 'positive', 'threatening', 'saving', 'type', 'wide', 'born', 'being', 'spectrum', 'onset',
]);
function isLower(code: number): boolean {
return code >= 97 && code <= 122;
}
function isLetter(code: number): boolean {
return (code >= 97 && code <= 122) || (code >= 65 && code <= 90);
}
/** Re-joins "medica-\ntion" -> "medication", leaving likely real compounds alone. */
function joinHyphenatedLineBreaks(text: string): string {
if (!text.includes('-\n')) return text;
return text.replace(/-\n[ \t]*(?=[a-z]{2})/g, (match: string, offset: number, whole: string) => {
// Left fragment: lowercase letters, optionally led by one capital, at most 30 long.
let start = offset;
while (start > 0 && offset - start < 30 && isLower(whole.charCodeAt(start - 1))) start--;
if (start > 0 && offset - start < 30 && isLetter(whole.charCodeAt(start - 1))) start--;
const left = whole.slice(start, offset);
if (left.length < 2) return match;
if (start > 0) {
const before = whole.charCodeAt(start - 1);
// Part of a longer token ("state-of-the-\nart", "ABCdef-\n...", "x2-\n..."): leave it.
if (before === 45 || isLetter(before) || (before >= 48 && before <= 57)) return match;
}
const rightStart = offset + match.length;
let rightEnd = rightStart;
while (rightEnd < whole.length && rightEnd - rightStart < 30 && isLower(whole.charCodeAt(rightEnd))) rightEnd++;
const right = whole.slice(rightStart, rightEnd);
if (rightEnd < whole.length && whole.charCodeAt(rightEnd) === 45) return match; // "x-\ny-z"
if (HYPHEN_KEEP_LEFT.has(left.toLowerCase()) || HYPHEN_KEEP_RIGHT.has(right)) return match;
return '';
});
}
/**
* Normalises extracted text: newlines, unicode spaces, control characters,
* ligatures, trailing spaces, runs of blank lines and words hyphenated across
* a line break. Idempotent.
*/
export function cleanText(text: string): string {
if (!text) return '';
let t = text;
if (t.includes('\r')) t = t.replace(/\r\n?/g, '\n');
t = t.replace(/[\u2028\u2029\f\v]/g, '\n');
t = t.replace(/[\u00A0\u1680\u2000-\u200A\u202F\u205F\u3000]/g, ' ');
// NUL, C0/C1 controls (tab and newline survive), soft hyphen, zero-width space, word joiner, BOM.
t = t.replace(/[\u0000-\u0008\u000E-\u001F\u007F-\u009F\u00AD\u200B\u2060\uFEFF\uFFFE\uFFFF]/g, '');
t = t.replace(/[\uFB00-\uFB06]/g, (ch) => LIGATURES[ch] ?? ch);
// Trailing whitespace per line (done by splitting so a huge run of spaces stays linear).
t = t
.split('\n')
.map((line) => line.trimEnd())
.join('\n');
t = joinHyphenatedLineBreaks(t);
t = t.replace(/\n{3,}/g, '\n\n');
return t.trim();
}
// ---------------------------------------------------------------------------
// Repeated page headers / footers
// ---------------------------------------------------------------------------
function edgeLineKey(line: string): string | null {
const trimmed = line.trim();
if (!trimmed || trimmed.length > 100) return null;
return trimmed.toLowerCase().replace(/\d+/g, '#').replace(/\s+/g, ' ');
}
/**
* Removes running headers/footers and page numbers from per-page text, but only
* when the same first or last line (digits ignored) shows up on at least 60% of
* the pages that have text, and there are at least four such pages. Chapter
* headings that change through a book are therefore left alone.
*/
export function stripRepeatedPageLines(pages: string[]): string[] {
let current = pages.map((page) => page.split('\n'));
for (let pass = 0; pass < 2; pass++) {
const firstIdx: number[] = [];
const lastIdx: number[] = [];
const topCounts = new Map<string, number>();
const bottomCounts = new Map<string, number>();
let textPages = 0;
for (let p = 0; p < current.length; p++) {
const lines = current[p];
let first = 0;
while (first < lines.length && !lines[first].trim()) first++;
let last = lines.length - 1;
while (last >= 0 && !lines[last].trim()) last--;
firstIdx.push(first < lines.length ? first : -1);
lastIdx.push(last);
if (last < 0) continue;
textPages++;
const top = edgeLineKey(lines[first]);
if (top) topCounts.set(top, (topCounts.get(top) ?? 0) + 1);
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom) bottomCounts.set(bottom, (bottomCounts.get(bottom) ?? 0) + 1);
}
}
if (textPages < 4) break;
const threshold = Math.max(3, Math.ceil(textPages * 0.6));
let removed = false;
current = current.map((lines, p) => {
const first = firstIdx[p];
const last = lastIdx[p];
if (last < 0) return lines;
let from = 0;
let to = lines.length;
const top = edgeLineKey(lines[first]);
if (top && (topCounts.get(top) ?? 0) >= threshold) from = first + 1;
if (last !== first) {
const bottom = edgeLineKey(lines[last]);
if (bottom && (bottomCounts.get(bottom) ?? 0) >= threshold) to = last;
}
if (from === 0 && to === lines.length) return lines;
removed = true;
return lines.slice(from, to);
});
if (!removed) break;
}
return current.map((lines) => lines.join('\n'));
}
// ---------------------------------------------------------------------------
// Chunking
// ---------------------------------------------------------------------------
/** A piece of a section that is never cut. `sep`/`strength` describe the boundary before it. */
interface Atom {
text: string;
raw: number;
sep: string;
sepRaw: number;
/** 4 new heading, 3 paragraph, 2 list item / line after a full stop, 1 sentence, 0.5 soft line wrap, 0 word, -1 inside a word. */
strength: number;
}
const WHITESPACE_RAW = 0.25;
/** Sentence end: punctuation, optional closing quote/bracket, spaces, then something that looks like a start. */
const SENTENCE_BOUNDARY = /(?<=[.!?]["'”’)\]]?)[ \t]+(?=["'“‘(\[]?[A-Z0-9])/;
const HEADING_LINE = /^ {0,3}#{1,6}[ \t]/;
const BLOCK_START_LINE = /^[ \t]*(?:[-*+•●◦▪–]|\d{1,3}[.)]|[a-zA-Z][.)]|\|)[ \t]?/;
const SENTENCE_END_LINE = /[.!?:;]["'”’)\]]?$/;
function sepRawOf(sep: string): number {
return sep ? WHITESPACE_RAW : 0;
}
/** Cuts an unbroken string into slices of at most `limit` estimated tokens, never inside a surrogate pair. */
function hardSplit(word: string, limit: number, out: Atom[], sep: string, strength: number): void {
let start = 0;
let acc = 0;
let first = true;
const push = (end: number) => {
const text = word.slice(start, end);
out.push({
text,
raw: rawTokens(text),
sep: first ? sep : '',
sepRaw: first ? sepRawOf(sep) : 0,
strength: first ? strength : -1,
});
first = false;
start = end;
acc = 0;
};
for (let i = 0; i < word.length; i++) {
const c = word.charCodeAt(i);
const isLowSurrogate = c >= 0xdc00 && c <= 0xdfff;
const weight = c < 128 ? 0.5 : 1; // upper bound of the per-character cost
if (acc + weight > limit && i > start && !isLowSurrogate) push(i);
acc += weight;
}
if (start < word.length) push(word.length);
}
function pushSentence(sentence: string, atomMax: number, out: Atom[], sep: string, strength: number): void {
const raw = rawTokens(sentence);
if (raw <= atomMax) {
out.push({ text: sentence, raw, sep, sepRaw: sepRawOf(sep), strength });
return;
}
// Oversized sentence (or table row, or blob): fall back to groups of words.
const groupMax = Math.max(1, Math.min(atomMax, 24));
const words = sentence.split(/[ \t]+/);
// Keep leading indentation attached to the first word.
const indent = /^[ \t]+/.exec(sentence)?.[0] ?? '';
let group = '';
let groupRaw = 0;
let first = true;
const flush = () => {
if (!group) return;
out.push({
text: group,
raw: rawTokens(group),
sep: first ? sep : ' ',
sepRaw: first ? sepRawOf(sep) : WHITESPACE_RAW,
strength: first ? strength : 0,
});
first = false;
group = '';
groupRaw = 0;
};
let pendingIndent = indent;
for (const word of words) {
if (!word) continue;
const piece = pendingIndent + word;
pendingIndent = '';
const wordRaw = rawTokens(piece);
if (wordRaw > atomMax) {
flush();
hardSplit(piece, atomMax, out, first ? sep : ' ', first ? strength : 0);
first = false;
continue;
}
if (group && groupRaw + WHITESPACE_RAW + wordRaw > groupMax) flush();
group = group ? `${group} ${piece}` : piece;
groupRaw += wordRaw + (groupRaw ? WHITESPACE_RAW : 0);
}
flush();
}
/** Breaks a section into atoms: paragraphs -> lines -> sentences -> word groups -> character slices. */
function atomize(text: string, atomMax: number): Atom[] {
const atoms: Atom[] = [];
let prevLine = '';
for (const para of text.split(/\n{2,}/)) {
let firstInPara = true;
for (const rawLine of para.split('\n')) {
const line = rawLine.trimEnd();
if (!line.trim()) continue;
let sep = '';
let strength = 0;
if (atoms.length > 0) {
const afterHeading = HEADING_LINE.test(prevLine);
const isHeading = HEADING_LINE.test(line);
if (firstInPara) {
sep = '\n\n';
strength = isHeading ? 4 : afterHeading ? -0.5 : 3;
} else {
sep = '\n';
if (isHeading) strength = 4;
else if (afterHeading) strength = -0.5;
else if (BLOCK_START_LINE.test(line) || SENTENCE_END_LINE.test(prevLine)) strength = 2;
else strength = 0.5;
}
}
firstInPara = false;
prevLine = line;
if (rawTokens(line) <= Math.min(atomMax, 12)) {
// Short line: not worth looking for sentences.
pushSentence(line, atomMax, atoms, sep, strength);
continue;
}
let firstSentence = true;
for (const sentence of line.split(SENTENCE_BOUNDARY)) {
if (!sentence) continue;
pushSentence(sentence, atomMax, atoms, firstSentence ? sep : ' ', firstSentence ? strength : 1);
firstSentence = false;
}
}
}
return atoms;
}
function joinAtoms(atoms: Atom[], from: number, to: number): string {
let text = '';
for (let i = from; i < to; i++) text += i === from ? atoms[i].text : atoms[i].sep + atoms[i].text;
return text;
}
/** Trailing whole words of `text` worth at most `budget` tokens. */
function tailWords(text: string, budget: number): string {
let end = text.length;
let cut = text.length;
let acc = 0;
while (end > 0) {
let wordEnd = end;
while (wordEnd > 0 && text.charCodeAt(wordEnd - 1) === 32) wordEnd--;
if (wordEnd === 0) break;
const space = text.lastIndexOf(' ', wordEnd - 1);
const word = text.slice(space + 1, wordEnd);
const cost = rawTokens(word) + WHITESPACE_RAW;
if (acc + cost > budget) break;
acc += cost;
cut = space + 1;
end = space < 0 ? 0 : space;
}
return text.slice(cut).trim();
}
/**
* Splits one oversized section into balanced pieces of at most `bodyBudget`
* tokens, cutting at the strongest boundary near the balanced size, and
* prefixing each later piece with up to `overlap` tokens of trailing context.
*/
function splitSection(text: string, bodyBudget: number, overlap: number): string[] {
const atomMax = Math.max(1, bodyBudget / 4);
const atoms = atomize(text, atomMax);
const n = atoms.length;
const pieces: string[] = [];
if (n === 0) return pieces;
// suffix[i] = tokens of atoms i..n-1 including the separators before each of them.
const suffix = new Float64Array(n + 1);
for (let i = n - 1; i >= 0; i--) suffix[i] = suffix[i + 1] + atoms[i].raw + atoms[i].sepRaw;
let start = 0;
let prevStart = -1;
while (start < n) {
const remaining = suffix[start] - atoms[start].sepRaw;
let end: number;
if (remaining <= bodyBudget) {
end = n;
} else {
const goal = remaining / Math.ceil(remaining / bodyBudget);
const minFill = goal * 0.6;
let acc = 0;
let best = -1;
let bestStrength = -Infinity;
let bestDistance = Infinity;
let i = start;
for (; i < n; i++) {
const add = atoms[i].raw + (i > start ? atoms[i].sepRaw : 0);
if (i > start && acc + add > bodyBudget) break;
acc += add;
if (i + 1 < n && acc >= minFill) {
const strength = atoms[i + 1].strength;
const distance = Math.abs(acc - goal);
if (strength > bestStrength || (strength === bestStrength && distance < bestDistance)) {
best = i;
bestStrength = strength;
bestDistance = distance;
}
}
}
end = best >= 0 ? best + 1 : Math.max(start + 1, i);
}
let prefix = '';
if (prevStart >= 0 && overlap > 0 && atoms[start].strength < 4) {
// Trailing context from the previous piece: whole atoms if they fit, else trailing words.
let from = start;
let acc = 0;
while (from - 1 > prevStart) {
const candidate = atoms[from - 1];
const cost = candidate.raw + (from < start ? atoms[from].sepRaw : 0);
if (acc + cost > overlap) break;
acc += cost;
from--;
if (candidate.strength >= 4) break; // do not reach back past a heading
}
if (from < start) prefix = joinAtoms(atoms, from, start);
else if (atoms[start].strength >= 0) prefix = tailWords(atoms[start - 1].text, overlap);
}
const body = joinAtoms(atoms, start, end);
pieces.push(prefix ? prefix + atoms[start].sep + body : body);
prevStart = start;
start = end;
}
return pieces;
}
function positiveInt(value: number | undefined, fallback: number, min: number): number {
if (typeof value !== 'number' || !Number.isFinite(value)) return fallback;
return Math.max(min, Math.floor(value));
}
interface Pending {
text: string;
heading: string;
page: number | null;
}
/**
* Builds retrieval chunks from the sections of ONE document.
*
* - Consecutive small sections (short slides, short pages) are packed together
* up to `targetTokens`; the chunk keeps the first section's heading and the
* full page range.
* - A section that fits comfortably under the ceiling stays whole so chunks
* line up with pages/slides.
* - Larger sections are split into balanced pieces at paragraph, then
* sentence, then word boundaries, with `overlapTokens` of repeated context
* between consecutive pieces of the same section only.
*/
export function chunkSections(sections: ExtractedSection[], opts: ChunkOptions = {}): DocChunk[] {
const target = positiveInt(opts.targetTokens, 450, 8);
const max = Math.max(target, positiveInt(opts.maxTokens, 700, 8));
const overlap = Math.min(positiveInt(opts.overlapTokens, 60, 0), Math.floor(target / 2));
/** Sections up to this size are kept in one piece. */
const wholeLimit = Math.floor(target + (max - target) / 2);
/** Budget for the non-overlap part of a split piece; leaves room for the overlap under `max`. */
const bodyBudget = Math.max(2, Math.min(target, max - overlap - 1));
const chunks: DocChunk[] = [];
const emit = (text: string, heading: string, pageStart: number | null, pageEnd: number | null) => {
const trimmed = text.trim();
if (!trimmed) return;
chunks.push({ seq: chunks.length, text: trimmed, heading, pageStart, pageEnd, tokens: estimateTokens(trimmed) });
};
let pending: Pending[] = [];
let pendingRaw = 0;
const flush = () => {
if (pending.length === 0) return;
let pageStart: number | null = null;
let pageEnd: number | null = null;
for (const item of pending) {
if (item.page === null) continue;
pageStart = pageStart === null ? item.page : Math.min(pageStart, item.page);
pageEnd = pageEnd === null ? item.page : Math.max(pageEnd, item.page);
}
emit(pending.map((item) => item.text).join('\n\n'), pending[0].heading, pageStart, pageEnd);
pending = [];
pendingRaw = 0;
};
let lastHeading = '';
for (const section of sections ?? []) {
if (!section || typeof section.text !== 'string') continue;
const ownHeading = typeof section.heading === 'string' ? section.heading.replace(/\s+/g, ' ').trim().slice(0, 200) : '';
if (ownHeading) lastHeading = ownHeading;
const text = (section.text.includes('\r') ? section.text.replace(/\r\n?/g, '\n') : section.text).trim();
if (!text) continue;
const heading = lastHeading;
const page = typeof section.page === 'number' && Number.isFinite(section.page) ? section.page : null;
const raw = rawTokens(text);
if (raw > wholeLimit) {
flush();
for (const piece of splitSection(text, bodyBudget, overlap)) emit(piece, heading, page, page);
continue;
}
if (pending.length > 0 && pendingRaw + WHITESPACE_RAW + raw > target) flush();
pendingRaw += raw + (pending.length > 0 ? WHITESPACE_RAW : 0);
pending.push({ text, heading, page });
}
flush();
return chunks;
}