Start Study Desk V2: remove V1 sources, add backend foundation
V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
7bd0a9aad4
commit
3e74ecd843
268 changed files with 17740 additions and 55423 deletions
272
server/documents/markup.ts
Normal file
272
server/documents/markup.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
/** Small markup helpers shared by the DOCX and PPTX extractors. */
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: '&',
|
||||
lt: '<',
|
||||
gt: '>',
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
nbsp: ' ',
|
||||
};
|
||||
|
||||
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
|
||||
export function decodeEntities(text: string): string {
|
||||
if (!text.includes('&')) return text;
|
||||
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
|
||||
if (body[0] === '#') {
|
||||
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
||||
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
|
||||
return String.fromCodePoint(code);
|
||||
}
|
||||
return NAMED_ENTITIES[body] ?? match;
|
||||
});
|
||||
}
|
||||
|
||||
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
|
||||
export function renderTable(rows: string[][]): string {
|
||||
const cleaned = rows
|
||||
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
|
||||
.filter((row) => row.some((cell) => cell.length > 0));
|
||||
if (cleaned.length === 0) return '';
|
||||
const width = Math.max(...cleaned.map((row) => row.length));
|
||||
const line = (row: string[]) => {
|
||||
const cells = row.slice();
|
||||
while (cells.length < width) cells.push('');
|
||||
return `| ${cells.join(' | ')} |`;
|
||||
};
|
||||
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
|
||||
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTML (as produced by mammoth) -> Markdown blocks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface MarkdownBlock {
|
||||
/** 1-6 for headings, 0 for everything else. */
|
||||
level: number;
|
||||
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
|
||||
text: string;
|
||||
}
|
||||
|
||||
interface HtmlElement {
|
||||
tag: string;
|
||||
children: HtmlNode[];
|
||||
}
|
||||
type HtmlNode = HtmlElement | string;
|
||||
|
||||
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
|
||||
const HEADING_TAG = /^h[1-6]$/;
|
||||
|
||||
function parseHtml(html: string): HtmlElement {
|
||||
const root: HtmlElement = { tag: '#root', children: [] };
|
||||
const stack: HtmlElement[] = [root];
|
||||
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = tokens.exec(html)) !== null) {
|
||||
const top = stack[stack.length - 1];
|
||||
if (match[4] !== undefined) {
|
||||
top.children.push(decodeEntities(match[4]));
|
||||
} else if (match[2] === undefined) {
|
||||
top.children.push('<');
|
||||
} else {
|
||||
const tag = match[2].toLowerCase();
|
||||
if (match[1]) {
|
||||
// Close the nearest matching open element; ignore stray closers.
|
||||
for (let i = stack.length - 1; i > 0; i--) {
|
||||
if (stack[i].tag === tag) {
|
||||
stack.length = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const element: HtmlElement = { tag, children: [] };
|
||||
top.children.push(element);
|
||||
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function collapse(text: string): string {
|
||||
return text.replace(/[ \t\r\n]+/g, ' ');
|
||||
}
|
||||
|
||||
function wrap(marker: string, inner: string): string {
|
||||
const core = inner.trim();
|
||||
if (!core) return inner;
|
||||
const lead = inner.slice(0, inner.length - inner.trimStart().length);
|
||||
const trail = inner.slice(inner.trimEnd().length);
|
||||
return `${lead}${marker}${core}${marker}${trail}`;
|
||||
}
|
||||
|
||||
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
|
||||
function inlineText(nodes: HtmlNode[], plain: boolean): string {
|
||||
let out = '';
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') {
|
||||
out += collapse(node);
|
||||
continue;
|
||||
}
|
||||
switch (node.tag) {
|
||||
case 'br':
|
||||
out += '\n';
|
||||
break;
|
||||
case 'img':
|
||||
case 'hr':
|
||||
break;
|
||||
case 'strong':
|
||||
case 'b':
|
||||
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
|
||||
break;
|
||||
case 'em':
|
||||
case 'i':
|
||||
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
|
||||
break;
|
||||
case 'a': {
|
||||
const text = inlineText(node.children, plain);
|
||||
// mammoth's footnote back-references
|
||||
if (text.trim() !== '↑') out += text;
|
||||
break;
|
||||
}
|
||||
case 'p':
|
||||
case 'li':
|
||||
case 'tr':
|
||||
case 'td':
|
||||
case 'th':
|
||||
case 'ul':
|
||||
case 'ol':
|
||||
case 'table': {
|
||||
const inner = inlineText(node.children, plain).trim();
|
||||
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
|
||||
break;
|
||||
}
|
||||
default:
|
||||
out += inlineText(node.children, plain);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function tidyInline(text: string): string {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/ {2,}/g, ' ').trim())
|
||||
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
|
||||
const ordered = list.tag === 'ol';
|
||||
let counter = 0;
|
||||
for (const item of list.children) {
|
||||
if (typeof item === 'string' || item.tag !== 'li') continue;
|
||||
counter++;
|
||||
const marker = ordered ? `${counter}. ` : '- ';
|
||||
const inlineNodes: HtmlNode[] = [];
|
||||
const nested: HtmlElement[] = [];
|
||||
const paragraphs: string[] = [];
|
||||
const flushInline = () => {
|
||||
const text = tidyInline(inlineText(inlineNodes, false));
|
||||
if (text) paragraphs.push(text);
|
||||
inlineNodes.length = 0;
|
||||
};
|
||||
for (const child of item.children) {
|
||||
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
|
||||
flushInline();
|
||||
nested.push(child);
|
||||
} else if (typeof child !== 'string' && child.tag === 'p') {
|
||||
flushInline();
|
||||
inlineNodes.push(...child.children);
|
||||
flushInline();
|
||||
} else if (typeof child !== 'string' && child.tag === 'table') {
|
||||
flushInline();
|
||||
const table = renderHtmlTable(child);
|
||||
if (table) paragraphs.push(table);
|
||||
} else {
|
||||
inlineNodes.push(child);
|
||||
}
|
||||
}
|
||||
flushInline();
|
||||
const pad = indent + ' '.repeat(marker.length);
|
||||
const body = paragraphs.join('\n').split('\n');
|
||||
if (body.length > 0 && body[0] !== '') {
|
||||
lines.push(indent + marker + body[0]);
|
||||
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
|
||||
} else if (nested.length === 0) {
|
||||
continue;
|
||||
}
|
||||
for (const child of nested) renderList(child, pad, lines);
|
||||
}
|
||||
}
|
||||
|
||||
function renderHtmlTable(table: HtmlElement): string {
|
||||
const rows: string[][] = [];
|
||||
const visit = (nodes: HtmlNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') continue;
|
||||
if (node.tag === 'tr') {
|
||||
const cells: string[] = [];
|
||||
for (const cell of node.children) {
|
||||
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
|
||||
cells.push(tidyInline(inlineText(cell.children, false)));
|
||||
}
|
||||
rows.push(cells);
|
||||
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
|
||||
visit(node.children);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(table.children);
|
||||
return renderTable(rows);
|
||||
}
|
||||
|
||||
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
|
||||
export function htmlToBlocks(html: string): MarkdownBlock[] {
|
||||
const root = parseHtml(html);
|
||||
const blocks: MarkdownBlock[] = [];
|
||||
const loose: HtmlNode[] = [];
|
||||
const flushLoose = () => {
|
||||
if (loose.length === 0) return;
|
||||
const text = tidyInline(inlineText(loose, false));
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
loose.length = 0;
|
||||
};
|
||||
const visit = (nodes: HtmlNode[]) => {
|
||||
for (const node of nodes) {
|
||||
if (typeof node === 'string') {
|
||||
loose.push(node);
|
||||
continue;
|
||||
}
|
||||
if (HEADING_TAG.test(node.tag)) {
|
||||
flushLoose();
|
||||
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
|
||||
if (text) blocks.push({ level: Number(node.tag[1]), text });
|
||||
} else if (node.tag === 'p') {
|
||||
flushLoose();
|
||||
const text = tidyInline(inlineText(node.children, false));
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
} else if (node.tag === 'ul' || node.tag === 'ol') {
|
||||
flushLoose();
|
||||
const lines: string[] = [];
|
||||
renderList(node, '', lines);
|
||||
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
|
||||
} else if (node.tag === 'table') {
|
||||
flushLoose();
|
||||
const text = renderHtmlTable(node);
|
||||
if (text) blocks.push({ level: 0, text });
|
||||
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
|
||||
flushLoose();
|
||||
visit(node.children);
|
||||
} else {
|
||||
loose.push(node);
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(root.children);
|
||||
flushLoose();
|
||||
return blocks;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue