V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
424 lines
14 KiB
TypeScript
424 lines
14 KiB
TypeScript
/** PPTX extraction: one section per slide, in presentation order, with titles, bullets, tables and notes. */
|
|
import { XMLParser } from 'fast-xml-parser';
|
|
import { cleanText } from './chunk.ts';
|
|
import { decodeEntities, renderTable } from './markup.ts';
|
|
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
import { ZipReader, type ZipLimits } from './zip.ts';
|
|
|
|
export interface OfficeContext {
|
|
limits: ZipLimits;
|
|
onProgress?: (done: number, total: number) => void;
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
type XmlNode = Record<string, unknown>;
|
|
|
|
const parser = new XMLParser({
|
|
ignoreAttributes: false,
|
|
attributeNamePrefix: '',
|
|
preserveOrder: true,
|
|
trimValues: false,
|
|
parseTagValue: false,
|
|
parseAttributeValue: false,
|
|
// Entities are decoded by decodeEntities so numeric references work and nothing is expanded twice.
|
|
processEntities: false,
|
|
ignoreDeclaration: true,
|
|
ignorePiTags: true,
|
|
});
|
|
|
|
function parseXml(xml: string): XmlNode[] {
|
|
const parsed = parser.parse(xml) as unknown;
|
|
return Array.isArray(parsed) ? (parsed as XmlNode[]) : [];
|
|
}
|
|
|
|
function tagOf(node: XmlNode): string {
|
|
for (const key in node) if (key !== ':@') return key;
|
|
return '';
|
|
}
|
|
|
|
/** Element name without its namespace prefix. */
|
|
function localName(node: XmlNode): string {
|
|
const tag = tagOf(node);
|
|
const colon = tag.indexOf(':');
|
|
return colon < 0 ? tag : tag.slice(colon + 1);
|
|
}
|
|
|
|
function childrenOf(node: XmlNode): XmlNode[] {
|
|
const value = node[tagOf(node)];
|
|
return Array.isArray(value) ? (value as XmlNode[]) : [];
|
|
}
|
|
|
|
function attr(node: XmlNode, name: string): string | undefined {
|
|
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
|
if (!attrs) return undefined;
|
|
const direct = attrs[name];
|
|
if (typeof direct === 'string') return direct;
|
|
return undefined;
|
|
}
|
|
|
|
/** Attribute lookup that ignores the namespace prefix (r:id, rel:id, ...). */
|
|
function attrLocal(node: XmlNode, local: string): string | undefined {
|
|
const attrs = node[':@'] as Record<string, unknown> | undefined;
|
|
if (!attrs) return undefined;
|
|
for (const key in attrs) {
|
|
if (key.endsWith(`:${local}`) && typeof attrs[key] === 'string') return attrs[key] as string;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function child(node: XmlNode, local: string): XmlNode | undefined {
|
|
for (const c of childrenOf(node)) if (localName(c) === local) return c;
|
|
return undefined;
|
|
}
|
|
|
|
function findDeep(nodes: XmlNode[], local: string, depth = 0): XmlNode | undefined {
|
|
if (depth > 40) return undefined;
|
|
for (const node of nodes) {
|
|
if (localName(node) === local) return node;
|
|
const found = findDeep(childrenOf(node), local, depth + 1);
|
|
if (found) return found;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/** Concatenated text of every a:t / m:t descendant. */
|
|
function deepText(node: XmlNode, depth = 0): string {
|
|
if (depth > 40) return '';
|
|
const name = localName(node);
|
|
if (name === '#text') return '';
|
|
if (name === 't') {
|
|
let out = '';
|
|
for (const c of childrenOf(node)) if (typeof c['#text'] === 'string') out += c['#text'] as string;
|
|
return decodeEntities(out);
|
|
}
|
|
let out = '';
|
|
for (const c of childrenOf(node)) out += deepText(c, depth + 1);
|
|
return out;
|
|
}
|
|
|
|
interface Paragraph {
|
|
text: string;
|
|
level: number;
|
|
/** true: explicit bullet/number; false: bullets explicitly off; undefined: inherited. */
|
|
bullet: boolean | undefined;
|
|
}
|
|
|
|
function readParagraph(p: XmlNode): Paragraph {
|
|
let text = '';
|
|
let level = 0;
|
|
let bullet: boolean | undefined;
|
|
for (const c of childrenOf(p)) {
|
|
switch (localName(c)) {
|
|
case 'pPr': {
|
|
const lvl = Number(attr(c, 'lvl'));
|
|
if (Number.isFinite(lvl) && lvl > 0) level = Math.min(8, Math.floor(lvl));
|
|
for (const b of childrenOf(c)) {
|
|
const name = localName(b);
|
|
if (name === 'buNone') bullet = false;
|
|
else if (name === 'buChar' || name === 'buAutoNum' || name === 'buBlip') bullet = true;
|
|
}
|
|
break;
|
|
}
|
|
case 'br':
|
|
text += '\n';
|
|
break;
|
|
case 'fld': {
|
|
const type = attr(c, 'type') ?? '';
|
|
if (!/^(slidenum|datetime)/i.test(type)) text += deepText(c);
|
|
break;
|
|
}
|
|
case 'endParaRPr':
|
|
case '#text':
|
|
break;
|
|
default:
|
|
text += deepText(c);
|
|
}
|
|
}
|
|
text = text
|
|
.replace(/[\u000B\u2028]/g, '\n')
|
|
.split('\n')
|
|
.map((line) => line.replace(/[ \t\u00A0]+/g, ' ').trim())
|
|
.filter((line) => line.length > 0)
|
|
.join('\n');
|
|
return { text, level, bullet };
|
|
}
|
|
|
|
function readParagraphs(txBody: XmlNode): Paragraph[] {
|
|
const out: Paragraph[] = [];
|
|
for (const c of childrenOf(txBody)) {
|
|
if (localName(c) !== 'p') continue;
|
|
const paragraph = readParagraph(c);
|
|
if (paragraph.text) out.push(paragraph);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function renderParagraphs(paragraphs: Paragraph[], bulletsByDefault: boolean): string {
|
|
const lines: string[] = [];
|
|
for (const p of paragraphs) {
|
|
const bulleted = p.bullet ?? (bulletsByDefault || p.level > 0);
|
|
if (!bulleted) {
|
|
lines.push(p.text);
|
|
continue;
|
|
}
|
|
const indent = ' '.repeat(p.level);
|
|
const parts = p.text.split('\n');
|
|
lines.push(`${indent}- ${parts[0]}`);
|
|
for (let i = 1; i < parts.length; i++) lines.push(`${indent} ${parts[i]}`);
|
|
}
|
|
return lines.join('\n');
|
|
}
|
|
|
|
function readTable(tbl: XmlNode): string {
|
|
const rows: string[][] = [];
|
|
for (const tr of childrenOf(tbl)) {
|
|
if (localName(tr) !== 'tr') continue;
|
|
const cells: string[] = [];
|
|
for (const tc of childrenOf(tr)) {
|
|
if (localName(tc) !== 'tc') continue;
|
|
const txBody = child(tc, 'txBody');
|
|
cells.push(txBody ? readParagraphs(txBody).map((p) => p.text.replace(/\n/g, ' ')).join(' ') : '');
|
|
}
|
|
rows.push(cells);
|
|
}
|
|
return renderTable(rows);
|
|
}
|
|
|
|
interface SlideContent {
|
|
title: string;
|
|
blocks: string[];
|
|
}
|
|
|
|
const SKIPPED_PLACEHOLDERS = new Set(['sldNum', 'dt', 'ftr', 'hdr', 'sldImg']);
|
|
|
|
function placeholderType(shape: XmlNode): string | null {
|
|
const nv = child(shape, 'nvSpPr');
|
|
const nvPr = nv ? child(nv, 'nvPr') : undefined;
|
|
const ph = nvPr ? child(nvPr, 'ph') : undefined;
|
|
if (!ph) return null;
|
|
return attr(ph, 'type') ?? 'body';
|
|
}
|
|
|
|
function walkShapes(nodes: XmlNode[], into: SlideContent, depth: number): void {
|
|
if (depth > 30) return;
|
|
for (const node of nodes) {
|
|
switch (localName(node)) {
|
|
case 'sp': {
|
|
const type = placeholderType(node);
|
|
if (type && SKIPPED_PLACEHOLDERS.has(type)) break;
|
|
const txBody = child(node, 'txBody');
|
|
if (!txBody) break;
|
|
const paragraphs = readParagraphs(txBody);
|
|
if (paragraphs.length === 0) break;
|
|
if ((type === 'title' || type === 'ctrTitle') && !into.title) {
|
|
into.title = paragraphs.map((p) => p.text.replace(/\n/g, ' ')).join(' ').replace(/\s+/g, ' ').trim();
|
|
} else {
|
|
const bullets = type !== null && type !== 'subTitle' && type !== 'title' && type !== 'ctrTitle';
|
|
const text = renderParagraphs(paragraphs, bullets);
|
|
if (text) into.blocks.push(text);
|
|
}
|
|
break;
|
|
}
|
|
case 'grpSp':
|
|
walkShapes(childrenOf(node), into, depth + 1);
|
|
break;
|
|
case 'graphicFrame': {
|
|
const tbl = findDeep(childrenOf(node), 'tbl');
|
|
if (tbl) {
|
|
const table = readTable(tbl);
|
|
if (table) into.blocks.push(table);
|
|
}
|
|
break;
|
|
}
|
|
case 'AlternateContent': {
|
|
const choice = child(node, 'Choice') ?? child(node, 'Fallback');
|
|
if (choice) walkShapes(childrenOf(choice), into, depth + 1);
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
function readSlide(xml: string): SlideContent {
|
|
const content: SlideContent = { title: '', blocks: [] };
|
|
const spTree = findDeep(parseXml(xml), 'spTree');
|
|
if (spTree) walkShapes(childrenOf(spTree), content, 0);
|
|
return content;
|
|
}
|
|
|
|
function readNotes(xml: string): string {
|
|
const spTree = findDeep(parseXml(xml), 'spTree');
|
|
if (!spTree) return '';
|
|
const parts: string[] = [];
|
|
for (const shape of childrenOf(spTree)) {
|
|
if (localName(shape) !== 'sp' || placeholderType(shape) !== 'body') continue;
|
|
const txBody = child(shape, 'txBody');
|
|
if (!txBody) continue;
|
|
const text = renderParagraphs(readParagraphs(txBody), false);
|
|
if (text) parts.push(text);
|
|
}
|
|
return parts.join('\n');
|
|
}
|
|
|
|
interface Relationship {
|
|
id: string;
|
|
type: string;
|
|
target: string;
|
|
external: boolean;
|
|
}
|
|
|
|
function readRelationships(xml: string | null): Relationship[] {
|
|
if (!xml) return [];
|
|
const out: Relationship[] = [];
|
|
const tags = /<(?:[A-Za-z0-9]+:)?Relationship\b([^>]*)>/g;
|
|
let match: RegExpExecArray | null;
|
|
while ((match = tags.exec(xml)) !== null) {
|
|
const attrs: Record<string, string> = {};
|
|
const pairs = /([A-Za-z:]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
|
|
let pair: RegExpExecArray | null;
|
|
while ((pair = pairs.exec(match[1])) !== null) attrs[pair[1]] = decodeEntities(pair[2] ?? pair[3] ?? '');
|
|
if (attrs.Id && attrs.Target) {
|
|
out.push({ id: attrs.Id, type: attrs.Type ?? '', target: attrs.Target, external: attrs.TargetMode === 'External' });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Resolves a relationship target against the directory of the part that owns it. */
|
|
function resolveTarget(baseDir: string, target: string): string {
|
|
const parts = target.startsWith('/') ? [] : baseDir.split('/').filter(Boolean);
|
|
for (const segment of target.split('/')) {
|
|
if (!segment || segment === '.') continue;
|
|
if (segment === '..') parts.pop();
|
|
else parts.push(segment);
|
|
}
|
|
return parts.join('/');
|
|
}
|
|
|
|
function relsPathFor(part: string): string {
|
|
const slash = part.lastIndexOf('/');
|
|
return `${part.slice(0, slash + 1)}_rels/${part.slice(slash + 1)}.rels`;
|
|
}
|
|
|
|
function slideOrder(zip: ZipReader): string[] {
|
|
const ordered: string[] = [];
|
|
const presentation = zip.readText('ppt/presentation.xml');
|
|
if (presentation) {
|
|
const targets = new Map<string, string>();
|
|
for (const rel of readRelationships(zip.readText('ppt/_rels/presentation.xml.rels'))) {
|
|
if (!rel.external) targets.set(rel.id, resolveTarget('ppt', rel.target));
|
|
}
|
|
const list = findDeep(parseXml(presentation), 'sldIdLst');
|
|
if (list) {
|
|
for (const item of childrenOf(list)) {
|
|
if (localName(item) !== 'sldId') continue;
|
|
const id = attrLocal(item, 'id');
|
|
const path = id ? targets.get(id) : undefined;
|
|
if (path && zip.has(path) && !ordered.includes(path)) ordered.push(path);
|
|
}
|
|
}
|
|
}
|
|
if (ordered.length > 0) return ordered;
|
|
// Damaged or unusual package: fall back to numeric file order.
|
|
return zip
|
|
.names()
|
|
.map((name) => ({ name, match: /^ppt\/slides\/slide(\d+)\.xml$/i.exec(name) }))
|
|
.filter((item): item is { name: string; match: RegExpExecArray } => item.match !== null)
|
|
.sort((a, b) => Number(a.match[1]) - Number(b.match[1]))
|
|
.map((item) => item.name);
|
|
}
|
|
|
|
export function readCoreTitle(zip: ZipReader): string | undefined {
|
|
let xml: string | null = null;
|
|
try {
|
|
xml = zip.readText('docProps/core.xml');
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
if (!xml) return undefined;
|
|
const match = /<(?:[A-Za-z0-9]+:)?title\b[^>]*>([^<]*)</.exec(xml);
|
|
const title = match ? decodeEntities(match[1]).replace(/\s+/g, ' ').trim() : '';
|
|
return title || undefined;
|
|
}
|
|
|
|
export async function extractPptx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
|
const zip = new ZipReader(bytes, ctx.limits);
|
|
const slides = slideOrder(zip);
|
|
if (slides.length === 0) {
|
|
throw new ExtractionError('corrupt', "This PowerPoint file doesn't contain any slides we can read. Try saving it again and re-uploading.");
|
|
}
|
|
|
|
const sections: ExtractedSection[] = [];
|
|
const warnings: string[] = [];
|
|
let withoutText = 0;
|
|
let unreadable = 0;
|
|
let totalChars = 0;
|
|
let lastYield = Date.now();
|
|
|
|
for (let i = 0; i < slides.length; i++) {
|
|
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
|
const path = slides[i];
|
|
let text = '';
|
|
let title = '';
|
|
try {
|
|
const xml = zip.readText(path);
|
|
const slide = xml ? readSlide(xml) : { title: '', blocks: [] };
|
|
title = slide.title;
|
|
const parts: string[] = [];
|
|
if (slide.title) parts.push(`## ${slide.title}`);
|
|
parts.push(...slide.blocks);
|
|
|
|
const notesRel = readRelationships(zip.readText(relsPathFor(path))).find(
|
|
(rel) => !rel.external && /\/notesSlide$/.test(rel.type),
|
|
);
|
|
if (notesRel) {
|
|
const notesXml = zip.readText(resolveTarget(path.slice(0, path.lastIndexOf('/')), notesRel.target));
|
|
const notes = notesXml ? readNotes(notesXml) : '';
|
|
if (notes) parts.push(`Notes:\n${notes}`);
|
|
}
|
|
text = cleanText(parts.join('\n\n'));
|
|
} catch (error) {
|
|
if (error instanceof ExtractionError) throw error;
|
|
if ((error as { code?: string })?.code === 'too_large') throw error;
|
|
unreadable++;
|
|
}
|
|
|
|
if (text) {
|
|
totalChars += text.length;
|
|
const section: ExtractedSection = { text, page: i + 1 };
|
|
if (title) section.heading = title;
|
|
sections.push(section);
|
|
} else {
|
|
withoutText++;
|
|
}
|
|
ctx.onProgress?.(i + 1, slides.length);
|
|
if (Date.now() - lastYield > 20) {
|
|
await new Promise<void>((resolve) => setImmediate(resolve));
|
|
lastYield = Date.now();
|
|
}
|
|
}
|
|
|
|
const total = slides.length;
|
|
if (unreadable === total) {
|
|
throw new ExtractionError('corrupt', 'This PowerPoint file looks damaged and could not be read. Try saving it again and re-uploading.');
|
|
}
|
|
if (unreadable > 0) warnings.push(`${unreadable} of ${total} slides could not be read.`);
|
|
if (withoutText === total) {
|
|
warnings.push(`None of the ${total} slides had any text - the slides are probably images.`);
|
|
} else if (withoutText * 2 > total) {
|
|
warnings.push(`${withoutText} of ${total} slides had no text - most of this deck is probably images, which are not read.`);
|
|
}
|
|
|
|
const result: ExtractResult = {
|
|
kind: 'pptx',
|
|
sections,
|
|
pageCount: total,
|
|
warnings,
|
|
needsOcr: totalChars < total * 25,
|
|
};
|
|
const title = readCoreTitle(zip);
|
|
if (title) result.title = title;
|
|
return result;
|
|
}
|