Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

272
server/documents/markup.ts Normal file
View file

@ -0,0 +1,272 @@
/** Small markup helpers shared by the DOCX and PPTX extractors. */
const NAMED_ENTITIES: Record<string, string> = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
};
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
export function decodeEntities(text: string): string {
if (!text.includes('&')) return text;
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
if (body[0] === '#') {
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
return String.fromCodePoint(code);
}
return NAMED_ENTITIES[body] ?? match;
});
}
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
export function renderTable(rows: string[][]): string {
const cleaned = rows
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
.filter((row) => row.some((cell) => cell.length > 0));
if (cleaned.length === 0) return '';
const width = Math.max(...cleaned.map((row) => row.length));
const line = (row: string[]) => {
const cells = row.slice();
while (cells.length < width) cells.push('');
return `| ${cells.join(' | ')} |`;
};
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
return out.join('\n');
}
// ---------------------------------------------------------------------------
// HTML (as produced by mammoth) -> Markdown blocks
// ---------------------------------------------------------------------------
export interface MarkdownBlock {
/** 1-6 for headings, 0 for everything else. */
level: number;
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
text: string;
}
interface HtmlElement {
tag: string;
children: HtmlNode[];
}
type HtmlNode = HtmlElement | string;
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
const HEADING_TAG = /^h[1-6]$/;
function parseHtml(html: string): HtmlElement {
const root: HtmlElement = { tag: '#root', children: [] };
const stack: HtmlElement[] = [root];
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
let match: RegExpExecArray | null;
while ((match = tokens.exec(html)) !== null) {
const top = stack[stack.length - 1];
if (match[4] !== undefined) {
top.children.push(decodeEntities(match[4]));
} else if (match[2] === undefined) {
top.children.push('<');
} else {
const tag = match[2].toLowerCase();
if (match[1]) {
// Close the nearest matching open element; ignore stray closers.
for (let i = stack.length - 1; i > 0; i--) {
if (stack[i].tag === tag) {
stack.length = i;
break;
}
}
} else {
const element: HtmlElement = { tag, children: [] };
top.children.push(element);
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
}
}
}
return root;
}
function collapse(text: string): string {
return text.replace(/[ \t\r\n]+/g, ' ');
}
function wrap(marker: string, inner: string): string {
const core = inner.trim();
if (!core) return inner;
const lead = inner.slice(0, inner.length - inner.trimStart().length);
const trail = inner.slice(inner.trimEnd().length);
return `${lead}${marker}${core}${marker}${trail}`;
}
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
function inlineText(nodes: HtmlNode[], plain: boolean): string {
let out = '';
for (const node of nodes) {
if (typeof node === 'string') {
out += collapse(node);
continue;
}
switch (node.tag) {
case 'br':
out += '\n';
break;
case 'img':
case 'hr':
break;
case 'strong':
case 'b':
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
break;
case 'em':
case 'i':
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
break;
case 'a': {
const text = inlineText(node.children, plain);
// mammoth's footnote back-references
if (text.trim() !== '↑') out += text;
break;
}
case 'p':
case 'li':
case 'tr':
case 'td':
case 'th':
case 'ul':
case 'ol':
case 'table': {
const inner = inlineText(node.children, plain).trim();
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
break;
}
default:
out += inlineText(node.children, plain);
}
}
return out;
}
function tidyInline(text: string): string {
return text
.split('\n')
.map((line) => line.replace(/ {2,}/g, ' ').trim())
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
.join('\n')
.trim();
}
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
const ordered = list.tag === 'ol';
let counter = 0;
for (const item of list.children) {
if (typeof item === 'string' || item.tag !== 'li') continue;
counter++;
const marker = ordered ? `${counter}. ` : '- ';
const inlineNodes: HtmlNode[] = [];
const nested: HtmlElement[] = [];
const paragraphs: string[] = [];
const flushInline = () => {
const text = tidyInline(inlineText(inlineNodes, false));
if (text) paragraphs.push(text);
inlineNodes.length = 0;
};
for (const child of item.children) {
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
flushInline();
nested.push(child);
} else if (typeof child !== 'string' && child.tag === 'p') {
flushInline();
inlineNodes.push(...child.children);
flushInline();
} else if (typeof child !== 'string' && child.tag === 'table') {
flushInline();
const table = renderHtmlTable(child);
if (table) paragraphs.push(table);
} else {
inlineNodes.push(child);
}
}
flushInline();
const pad = indent + ' '.repeat(marker.length);
const body = paragraphs.join('\n').split('\n');
if (body.length > 0 && body[0] !== '') {
lines.push(indent + marker + body[0]);
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
} else if (nested.length === 0) {
continue;
}
for (const child of nested) renderList(child, pad, lines);
}
}
function renderHtmlTable(table: HtmlElement): string {
const rows: string[][] = [];
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') continue;
if (node.tag === 'tr') {
const cells: string[] = [];
for (const cell of node.children) {
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
cells.push(tidyInline(inlineText(cell.children, false)));
}
rows.push(cells);
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
visit(node.children);
}
}
};
visit(table.children);
return renderTable(rows);
}
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
export function htmlToBlocks(html: string): MarkdownBlock[] {
const root = parseHtml(html);
const blocks: MarkdownBlock[] = [];
const loose: HtmlNode[] = [];
const flushLoose = () => {
if (loose.length === 0) return;
const text = tidyInline(inlineText(loose, false));
if (text) blocks.push({ level: 0, text });
loose.length = 0;
};
const visit = (nodes: HtmlNode[]) => {
for (const node of nodes) {
if (typeof node === 'string') {
loose.push(node);
continue;
}
if (HEADING_TAG.test(node.tag)) {
flushLoose();
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
if (text) blocks.push({ level: Number(node.tag[1]), text });
} else if (node.tag === 'p') {
flushLoose();
const text = tidyInline(inlineText(node.children, false));
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'ul' || node.tag === 'ol') {
flushLoose();
const lines: string[] = [];
renderList(node, '', lines);
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
} else if (node.tag === 'table') {
flushLoose();
const text = renderHtmlTable(node);
if (text) blocks.push({ level: 0, text });
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
flushLoose();
visit(node.children);
} else {
loose.push(node);
}
}
};
visit(root.children);
flushLoose();
return blocks;
}