V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
105 lines
4.2 KiB
TypeScript
105 lines
4.2 KiB
TypeScript
/** DOCX extraction through mammoth: HTML -> Markdown-ish text, split into sections at H1-H3. */
|
|
import { zipSync } from 'fflate';
|
|
import mammoth from 'mammoth';
|
|
import { cleanText } from './chunk.ts';
|
|
import { htmlToBlocks } from './markup.ts';
|
|
import { readCoreTitle, type OfficeContext } from './pptx.ts';
|
|
import { ExtractionError, type ExtractedSection, type ExtractResult } from './types.ts';
|
|
import { ZipReader } from './zip.ts';
|
|
|
|
const STYLE_MAP = ["p[style-name='Title'] => h1:fresh", "p[style-name='Subtitle'] => p:fresh"];
|
|
const MAX_WARNINGS = 3;
|
|
|
|
/**
|
|
* Re-packs only the XML parts mammoth reads (no media, fonts or embeddings)
|
|
* into a fresh stored zip. Every byte went through the size-limited reader, so
|
|
* mammoth's own unzip can never be handed a bomb, and it never spends time
|
|
* base64-encoding images we would throw away.
|
|
*/
|
|
function sanitisedPackage(zip: ZipReader): { data: Uint8Array; images: number } {
|
|
const files: Record<string, Uint8Array> = {};
|
|
let images = 0;
|
|
for (const entry of zip.entries) {
|
|
const name = entry.name;
|
|
if (name.endsWith('/')) continue;
|
|
if (/^word\/media\//i.test(name)) {
|
|
images++;
|
|
continue;
|
|
}
|
|
if (!/\.(xml|rels)$/i.test(name)) continue;
|
|
if (/^(customXml|docProps\/thumbnail)/i.test(name)) continue;
|
|
const data = zip.read(name);
|
|
if (data) files[name] = data;
|
|
}
|
|
return { data: zipSync(files, { level: 0 }), images };
|
|
}
|
|
|
|
function friendlyWarnings(messages: Array<{ type: string; message: string }>): string[] {
|
|
const seen = new Set<string>();
|
|
for (const item of messages) {
|
|
const message = String(item?.message ?? '').trim();
|
|
// Unknown style names never lose text; they are noise for a reader.
|
|
if (!message || /^Unrecognised (paragraph|run|table) style/i.test(message)) continue;
|
|
seen.add(message.length > 160 ? `${message.slice(0, 157)}...` : message);
|
|
}
|
|
const unique = [...seen];
|
|
const out = unique.slice(0, MAX_WARNINGS).map((message) => `Some content may be missing: ${message}`);
|
|
if (unique.length > MAX_WARNINGS) out.push(`...and ${unique.length - MAX_WARNINGS} more conversion notes.`);
|
|
return out;
|
|
}
|
|
|
|
export async function extractDocx(bytes: Uint8Array, ctx: OfficeContext): Promise<ExtractResult> {
|
|
const zip = new ZipReader(bytes, ctx.limits);
|
|
if (!zip.names().some((name) => /^word\/document\d*\.xml$/i.test(name))) {
|
|
throw new ExtractionError('corrupt', "This file doesn't look like a valid Word document. Try saving it again as .docx and re-uploading.");
|
|
}
|
|
const { data, images } = sanitisedPackage(zip);
|
|
if (ctx.signal?.aborted) throw new ExtractionError('failed', 'Reading this file was cancelled.', { cause: ctx.signal.reason });
|
|
|
|
const converted = await mammoth.convertToHtml(
|
|
{ buffer: Buffer.from(data.buffer, data.byteOffset, data.byteLength) },
|
|
{
|
|
styleMap: STYLE_MAP,
|
|
ignoreEmptyParagraphs: true,
|
|
// Images are not read locally; never decode them.
|
|
convertImage: mammoth.images.imgElement(async () => ({ src: '' })),
|
|
},
|
|
);
|
|
|
|
const sections: ExtractedSection[] = [];
|
|
let heading: string | undefined;
|
|
let parts: string[] = [];
|
|
const flush = () => {
|
|
const text = cleanText(parts.join('\n\n'));
|
|
parts = [];
|
|
if (!text) return;
|
|
const section: ExtractedSection = { text };
|
|
if (heading) section.heading = heading;
|
|
sections.push(section);
|
|
};
|
|
for (const block of htmlToBlocks(converted.value)) {
|
|
if (block.level >= 1 && block.level <= 3) {
|
|
flush();
|
|
heading = block.text.slice(0, 200);
|
|
}
|
|
parts.push(block.level > 0 ? `${'#'.repeat(block.level)} ${block.text}` : block.text);
|
|
}
|
|
flush();
|
|
ctx.onProgress?.(1, 1);
|
|
|
|
const warnings = friendlyWarnings(converted.messages ?? []);
|
|
const totalChars = sections.reduce((sum, section) => sum + section.text.length, 0);
|
|
let needsOcr = false;
|
|
if (totalChars < 25) {
|
|
if (images === 0) {
|
|
throw new ExtractionError('empty', "This Word document doesn't contain any text.");
|
|
}
|
|
needsOcr = true;
|
|
warnings.unshift(`This document has almost no text, only ${images} image${images === 1 ? '' : 's'} - pictures of text are not read.`);
|
|
}
|
|
|
|
const result: ExtractResult = { kind: 'docx', sections, warnings, needsOcr };
|
|
const title = readCoreTitle(zip);
|
|
if (title) result.title = title;
|
|
return result;
|
|
}
|