V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
272 lines
9 KiB
TypeScript
272 lines
9 KiB
TypeScript
/** Small markup helpers shared by the DOCX and PPTX extractors. */
|
|
|
|
const NAMED_ENTITIES: Record<string, string> = {
|
|
amp: '&',
|
|
lt: '<',
|
|
gt: '>',
|
|
quot: '"',
|
|
apos: "'",
|
|
nbsp: ' ',
|
|
};
|
|
|
|
/** Decodes the XML entities plus numeric character references. Unknown entities are left as written. */
|
|
export function decodeEntities(text: string): string {
|
|
if (!text.includes('&')) return text;
|
|
return text.replace(/&(#x[0-9a-fA-F]{1,6}|#[0-9]{1,7}|[a-zA-Z]{2,8});/g, (match, body: string) => {
|
|
if (body[0] === '#') {
|
|
const code = body[1] === 'x' || body[1] === 'X' ? parseInt(body.slice(2), 16) : parseInt(body.slice(1), 10);
|
|
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return '';
|
|
return String.fromCodePoint(code);
|
|
}
|
|
return NAMED_ENTITIES[body] ?? match;
|
|
});
|
|
}
|
|
|
|
/** Renders rows as a GitHub-style Markdown table, the first row being the header. */
|
|
export function renderTable(rows: string[][]): string {
|
|
const cleaned = rows
|
|
.map((row) => row.map((cell) => cell.replace(/\s*\n\s*/g, ' ').replace(/\|/g, '\\|').trim()))
|
|
.filter((row) => row.some((cell) => cell.length > 0));
|
|
if (cleaned.length === 0) return '';
|
|
const width = Math.max(...cleaned.map((row) => row.length));
|
|
const line = (row: string[]) => {
|
|
const cells = row.slice();
|
|
while (cells.length < width) cells.push('');
|
|
return `| ${cells.join(' | ')} |`;
|
|
};
|
|
const out = [line(cleaned[0]), `| ${Array.from({ length: width }, () => '---').join(' | ')} |`];
|
|
for (let i = 1; i < cleaned.length; i++) out.push(line(cleaned[i]));
|
|
return out.join('\n');
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// HTML (as produced by mammoth) -> Markdown blocks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface MarkdownBlock {
|
|
/** 1-6 for headings, 0 for everything else. */
|
|
level: number;
|
|
/** Markdown for the block. Headings carry their plain text here, without the leading #s. */
|
|
text: string;
|
|
}
|
|
|
|
interface HtmlElement {
|
|
tag: string;
|
|
children: HtmlNode[];
|
|
}
|
|
type HtmlNode = HtmlElement | string;
|
|
|
|
const VOID_TAGS = new Set(['br', 'img', 'hr', 'input', 'meta', 'link']);
|
|
const HEADING_TAG = /^h[1-6]$/;
|
|
|
|
function parseHtml(html: string): HtmlElement {
|
|
const root: HtmlElement = { tag: '#root', children: [] };
|
|
const stack: HtmlElement[] = [root];
|
|
const tokens = /<(\/?)([a-zA-Z][a-zA-Z0-9]*)[^>]*?(\/?)>|([^<]+)|</g;
|
|
let match: RegExpExecArray | null;
|
|
while ((match = tokens.exec(html)) !== null) {
|
|
const top = stack[stack.length - 1];
|
|
if (match[4] !== undefined) {
|
|
top.children.push(decodeEntities(match[4]));
|
|
} else if (match[2] === undefined) {
|
|
top.children.push('<');
|
|
} else {
|
|
const tag = match[2].toLowerCase();
|
|
if (match[1]) {
|
|
// Close the nearest matching open element; ignore stray closers.
|
|
for (let i = stack.length - 1; i > 0; i--) {
|
|
if (stack[i].tag === tag) {
|
|
stack.length = i;
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
const element: HtmlElement = { tag, children: [] };
|
|
top.children.push(element);
|
|
if (!match[3] && !VOID_TAGS.has(tag)) stack.push(element);
|
|
}
|
|
}
|
|
}
|
|
return root;
|
|
}
|
|
|
|
function collapse(text: string): string {
|
|
return text.replace(/[ \t\r\n]+/g, ' ');
|
|
}
|
|
|
|
function wrap(marker: string, inner: string): string {
|
|
const core = inner.trim();
|
|
if (!core) return inner;
|
|
const lead = inner.slice(0, inner.length - inner.trimStart().length);
|
|
const trail = inner.slice(inner.trimEnd().length);
|
|
return `${lead}${marker}${core}${marker}${trail}`;
|
|
}
|
|
|
|
/** Inline rendering: emphasis kept, links reduced to their text, block children flattened. */
|
|
function inlineText(nodes: HtmlNode[], plain: boolean): string {
|
|
let out = '';
|
|
for (const node of nodes) {
|
|
if (typeof node === 'string') {
|
|
out += collapse(node);
|
|
continue;
|
|
}
|
|
switch (node.tag) {
|
|
case 'br':
|
|
out += '\n';
|
|
break;
|
|
case 'img':
|
|
case 'hr':
|
|
break;
|
|
case 'strong':
|
|
case 'b':
|
|
out += plain ? inlineText(node.children, plain) : wrap('**', inlineText(node.children, plain));
|
|
break;
|
|
case 'em':
|
|
case 'i':
|
|
out += plain ? inlineText(node.children, plain) : wrap('*', inlineText(node.children, plain));
|
|
break;
|
|
case 'a': {
|
|
const text = inlineText(node.children, plain);
|
|
// mammoth's footnote back-references
|
|
if (text.trim() !== '↑') out += text;
|
|
break;
|
|
}
|
|
case 'p':
|
|
case 'li':
|
|
case 'tr':
|
|
case 'td':
|
|
case 'th':
|
|
case 'ul':
|
|
case 'ol':
|
|
case 'table': {
|
|
const inner = inlineText(node.children, plain).trim();
|
|
if (inner) out += (out && !/\s$/.test(out) ? ' ' : '') + inner + ' ';
|
|
break;
|
|
}
|
|
default:
|
|
out += inlineText(node.children, plain);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function tidyInline(text: string): string {
|
|
return text
|
|
.split('\n')
|
|
.map((line) => line.replace(/ {2,}/g, ' ').trim())
|
|
.filter((line, index, all) => line || (index > 0 && index < all.length - 1))
|
|
.join('\n')
|
|
.trim();
|
|
}
|
|
|
|
function renderList(list: HtmlElement, indent: string, lines: string[]): void {
|
|
const ordered = list.tag === 'ol';
|
|
let counter = 0;
|
|
for (const item of list.children) {
|
|
if (typeof item === 'string' || item.tag !== 'li') continue;
|
|
counter++;
|
|
const marker = ordered ? `${counter}. ` : '- ';
|
|
const inlineNodes: HtmlNode[] = [];
|
|
const nested: HtmlElement[] = [];
|
|
const paragraphs: string[] = [];
|
|
const flushInline = () => {
|
|
const text = tidyInline(inlineText(inlineNodes, false));
|
|
if (text) paragraphs.push(text);
|
|
inlineNodes.length = 0;
|
|
};
|
|
for (const child of item.children) {
|
|
if (typeof child !== 'string' && (child.tag === 'ul' || child.tag === 'ol')) {
|
|
flushInline();
|
|
nested.push(child);
|
|
} else if (typeof child !== 'string' && child.tag === 'p') {
|
|
flushInline();
|
|
inlineNodes.push(...child.children);
|
|
flushInline();
|
|
} else if (typeof child !== 'string' && child.tag === 'table') {
|
|
flushInline();
|
|
const table = renderHtmlTable(child);
|
|
if (table) paragraphs.push(table);
|
|
} else {
|
|
inlineNodes.push(child);
|
|
}
|
|
}
|
|
flushInline();
|
|
const pad = indent + ' '.repeat(marker.length);
|
|
const body = paragraphs.join('\n').split('\n');
|
|
if (body.length > 0 && body[0] !== '') {
|
|
lines.push(indent + marker + body[0]);
|
|
for (let i = 1; i < body.length; i++) lines.push(pad + body[i]);
|
|
} else if (nested.length === 0) {
|
|
continue;
|
|
}
|
|
for (const child of nested) renderList(child, pad, lines);
|
|
}
|
|
}
|
|
|
|
function renderHtmlTable(table: HtmlElement): string {
|
|
const rows: string[][] = [];
|
|
const visit = (nodes: HtmlNode[]) => {
|
|
for (const node of nodes) {
|
|
if (typeof node === 'string') continue;
|
|
if (node.tag === 'tr') {
|
|
const cells: string[] = [];
|
|
for (const cell of node.children) {
|
|
if (typeof cell === 'string' || (cell.tag !== 'td' && cell.tag !== 'th')) continue;
|
|
cells.push(tidyInline(inlineText(cell.children, false)));
|
|
}
|
|
rows.push(cells);
|
|
} else if (node.tag === 'thead' || node.tag === 'tbody' || node.tag === 'tfoot') {
|
|
visit(node.children);
|
|
}
|
|
}
|
|
};
|
|
visit(table.children);
|
|
return renderTable(rows);
|
|
}
|
|
|
|
/** Converts mammoth's HTML into a flat list of Markdown blocks (headings, paragraphs, lists, tables). */
|
|
export function htmlToBlocks(html: string): MarkdownBlock[] {
|
|
const root = parseHtml(html);
|
|
const blocks: MarkdownBlock[] = [];
|
|
const loose: HtmlNode[] = [];
|
|
const flushLoose = () => {
|
|
if (loose.length === 0) return;
|
|
const text = tidyInline(inlineText(loose, false));
|
|
if (text) blocks.push({ level: 0, text });
|
|
loose.length = 0;
|
|
};
|
|
const visit = (nodes: HtmlNode[]) => {
|
|
for (const node of nodes) {
|
|
if (typeof node === 'string') {
|
|
loose.push(node);
|
|
continue;
|
|
}
|
|
if (HEADING_TAG.test(node.tag)) {
|
|
flushLoose();
|
|
const text = tidyInline(inlineText(node.children, true)).replace(/\n+/g, ' ');
|
|
if (text) blocks.push({ level: Number(node.tag[1]), text });
|
|
} else if (node.tag === 'p') {
|
|
flushLoose();
|
|
const text = tidyInline(inlineText(node.children, false));
|
|
if (text) blocks.push({ level: 0, text });
|
|
} else if (node.tag === 'ul' || node.tag === 'ol') {
|
|
flushLoose();
|
|
const lines: string[] = [];
|
|
renderList(node, '', lines);
|
|
if (lines.length > 0) blocks.push({ level: 0, text: lines.join('\n') });
|
|
} else if (node.tag === 'table') {
|
|
flushLoose();
|
|
const text = renderHtmlTable(node);
|
|
if (text) blocks.push({ level: 0, text });
|
|
} else if (node.tag === 'div' || node.tag === 'section' || node.tag === 'blockquote' || node.tag === 'body') {
|
|
flushLoose();
|
|
visit(node.children);
|
|
} else {
|
|
loose.push(node);
|
|
}
|
|
}
|
|
};
|
|
visit(root.children);
|
|
flushLoose();
|
|
return blocks;
|
|
}
|