Start Study Desk V2: remove V1 sources, add backend foundation

V1 remains intact on main and in history. This branch replaces it with a new
application: node:sqlite storage with migrations, scrypt auth and sessions,
content/SRS/attempt/stats services, document extraction and chunking, a
provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible,
Ollama), the generation pipeline and the tool-using assistant.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-11-11 08:16:41 -07:00
parent 7bd0a9aad4
commit 3e74ecd843
268 changed files with 17740 additions and 55423 deletions

424
server/documents/pptx.ts Normal file
View file

@ -0,0 +1,424 @@
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
import { XMLParser } from 'fast-xml-parser';
import { cleanText } from './chunk.ts';
import { decodeEntities, renderTable } from './markup.ts';
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
import { ZipReader, type ZipLimits } from './zip.ts';
export interface OfficeContext {
limits: ZipLimits;
onProgress?: (done: number, total: number) => void;
signal?: AbortSignal;
}
type XmlNode = Record<string, unknown>;
const parser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '',
preserveOrder: true,
trimValues: false,
parseTagValue: false,
parseAttributeValue: false,
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
processEntities: false,
ignoreDeclaration: true,
ignorePiTags: true,
});
function parseXml(xml: string): XmlNode[] {
const parsed = parser.parse(xml) as unknown;
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
}
function tagOf(node: XmlNode): string {
for (const key in node) if (key !== ':@') return key;
return '';
}
/** Element name without its namespace prefix. */
function localName(node: XmlNode): string {
const tag = tagOf(node);
const colon = tag.indexOf(':');
return colon < 0 ? tag : tag.slice(colon + 1);
}
function childrenOf(node: XmlNode): XmlNode[] {
const value = node[tagOf(node)];
return Array.isArray(value) ? (value as XmlNode[]) : [];
}
function attr(node: XmlNode, name: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
const direct = attrs[name];
if (typeof direct === 'string') return direct;
return undefined;
}
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
function attrLocal(node: XmlNode, local: string): string | undefined {
const attrs = node[':@'] as Record<string, unknown> | undefined;
if (!attrs) return undefined;
for (const key in attrs) {
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
}
return undefined;
}
function child(node: XmlNode, local: string): XmlNode | undefined {
for (const c of childrenOf(node)) if (localName(c) === local) return c;
return undefined;
}
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
if (depth > 40) return undefined;
for (const node of nodes) {
if (localName(node) === local) return node;
const found = findDeep(childrenOf(node), local, depth + 1);
if (found) return found;
}
return undefined;
}
/** Concatenated text of every a:t / m:t descendant. */
function deepText(node: XmlNode, depth = 0): string {
if (depth > 40) return '';
const name = localName(node);
if (name === '#text') return '';
if (name === 't') {
let out = '';
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
return decodeEntities(out);
}
let out = '';
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
return out;
}
interface Paragraph {
text: string;
level: number;
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
bullet: boolean | undefined;
}
function readParagraph(p: XmlNode): Paragraph {
let text = '';
let level = 0;
let bullet: boolean | undefined;
for (const c of childrenOf(p)) {
switch (localName(c)) {
case 'pPr': {
const lvl = Number(attr(c, 'lvl'));
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
for (const b of childrenOf(c)) {
const name = localName(b);
if (name === 'buNone') bullet = false;
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
}
break;
}
case 'br':
text += '\n';
break;
case 'fld': {
const type = attr(c, 'type') ?? '';
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
break;
}
case 'endParaRPr':
case '#text':
break;
default:
text += deepText(c);
}
}
text = text
.replace(/[\u000B\u2028]/g, '\n')
.split('\n')
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
.filter((line) => line.length > 0)
.join('\n');
return { text, level, bullet };
}
function readParagraphs(txBody: XmlNode): Paragraph[] {
const out: Paragraph[] = [];
for (const c of childrenOf(txBody)) {
if (localName(c) !== 'p') continue;
const paragraph = readParagraph(c);
if (paragraph.text) out.push(paragraph);
}
return out;
}
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
const lines: string[] = [];
for (const p of paragraphs) {
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
if (!bulleted) {
lines.push(p.text);
continue;
}
const indent = ' '.repeat(p.level);
const parts = p.text.split('\n');
lines.push(`${indent}- ${parts[0]}`);
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
}
return lines.join('\n');
}
function readTable(tbl: XmlNode): string {
const rows: string[][] = [];
for (const tr of childrenOf(tbl)) {
if (localName(tr) !== 'tr') continue;
const cells: string[] = [];
for (const tc of childrenOf(tr)) {
if (localName(tc) !== 'tc') continue;
const txBody = child(tc, 'txBody');
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
}
rows.push(cells);
}
return renderTable(rows);
}
interface SlideContent {
title: string;
blocks: string[];
}
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
function placeholderType(shape: XmlNode): string | null {
const nv = child(shape, 'nvSpPr');
const nvPr = nv ? child(nv, 'nvPr') : undefined;
const ph = nvPr ? child(nvPr, 'ph') : undefined;
if (!ph) return null;
return attr(ph, 'type') ?? 'body';
}
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
if (depth > 30) return;
for (const node of nodes) {
switch (localName(node)) {
case 'sp': {
const type = placeholderType(node);
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
const txBody = child(node, 'txBody');
if (!txBody) break;
const paragraphs = readParagraphs(txBody);
if (paragraphs.length === 0) break;
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
} else {
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
const text = renderParagraphs(paragraphs, bullets);
if (text) into.blocks.push(text);
}
break;
}
case 'grpSp':
walkShapes(childrenOf(node), into, depth + 1);
break;
case 'graphicFrame': {
const tbl = findDeep(childrenOf(node), 'tbl');
if (tbl) {
const table = readTable(tbl);
if (table) into.blocks.push(table);
}
break;
}
case 'AlternateContent': {
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
break;
}
default:
break;
}
}
}
function readSlide(xml: string): SlideContent {
const content: SlideContent = { title: '', blocks: [] };
const spTree = findDeep(parseXml(xml), 'spTree');
if (spTree) walkShapes(childrenOf(spTree), content, 0);
return content;
}
function readNotes(xml: string): string {
const spTree = findDeep(parseXml(xml), 'spTree');
if (!spTree) return '';
const parts: string[] = [];
for (const shape of childrenOf(spTree)) {
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
const txBody = child(shape, 'txBody');
if (!txBody) continue;
const text = renderParagraphs(readParagraphs(txBody), false);
if (text) parts.push(text);
}
return parts.join('\n');
}
interface Relationship {
id: string;
type: string;
target: string;
external: boolean;
}
function readRelationships(xml: string | null): Relationship[] {
if (!xml) return [];
const out: Relationship[] = [];
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
let match: RegExpExecArray | null;
while ((match = tags.exec(xml)) !== null) {
const attrs: Record<string, string> = {};
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
let pair: RegExpExecArray | null;
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
if (attrs.Id && attrs.Target) {
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
}
}
return out;
}
/** Resolves a relationship target against the directory of the part that owns it. */
function resolveTarget(baseDir: string, target: string): string {
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
for (const segment of target.split('/')) {
if (!segment || segment === '.') continue;
if (segment === '..') parts.pop();
else parts.push(segment);
}
return parts.join('/');
}
function relsPathFor(part: string): string {
const slash = part.lastIndexOf('/');
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
}
function slideOrder(zip: ZipReader): string[] {
const ordered: string[] = [];
const presentation = zip.readText('ppt/presentation.xml');
if (presentation) {
const targets = new Map<string, string>();
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
}
const list = findDeep(parseXml(presentation), 'sldIdLst');
if (list) {
for (const item of childrenOf(list)) {
if (localName(item) !== 'sldId') continue;
const id = attrLocal(item, 'id');
const path = id ? targets.get(id) : undefined;
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
}
}
}
if (ordered.length > 0) return ordered;
// Damaged or unusual package: fall back to numeric file order.
return zip
.names()
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
.map((item) => item.name);
}
export function readCoreTitle(zip: ZipReader): string | undefined {
let xml: string | null = null;
try {
xml = zip.readText('docProps/core.xml');
} catch {
return undefined;
}
if (!xml) return undefined;
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
return title || undefined;
}
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
const zip = new ZipReader(bytes, ctx.limits);
const slides = slideOrder(zip);
if (slides.length === 0) {
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
}
const sections: ExtractedSection[] = [];
const warnings: string[] = [];
let withoutText = 0;
let unreadable = 0;
let totalChars = 0;
let lastYield = Date.now();
for (let i = 0; i < slides.length; i++) {
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
const path = slides[i];
let text = '';
let title = '';
try {
const xml = zip.readText(path);
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
title = slide.title;
const parts: string[] = [];
if (slide.title) parts.push(`## ${slide.title}`);
parts.push(...slide.blocks);
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
);
if (notesRel) {
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
const notes = notesXml ? readNotes(notesXml) : '';
if (notes) parts.push(`Notes:\n${notes}`);
}
text = cleanText(parts.join('\n\n'));
} catch (error) {
if (error instanceof ExtractionError) throw error;
if ((error as { code?: string })?.code === 'too_large') throw error;
unreadable++;
}
if (text) {
totalChars += text.length;
const section: ExtractedSection = { text, page: i + 1 };
if (title) section.heading = title;
sections.push(section);
} else {
withoutText++;
}
ctx.onProgress?.(i + 1, slides.length);
if (Date.now() - lastYield > 20) {
await new Promise<void>((resolve) => setImmediate(resolve));
lastYield = Date.now();
}
}
const total = slides.length;
if (unreadable === total) {
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
}
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
if (withoutText === total) {
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
} else if (withoutText * 2 > total) {
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
}
const result: ExtractResult = {
kind: 'pptx',
sections,
pageCount: total,
warnings,
needsOcr: totalChars < total * 25,
};
const title = readCoreTitle(zip);
if (title) result.title = title;
return result;
}