V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
270 lines
14 KiB
TypeScript
270 lines
14 KiB
TypeScript
/** Programmatic fixtures for the document tests. Nothing here is used by production code. */
|
|
import { strToU8, zipSync } from 'fflate';
|
|
|
|
const latin1 = (text: string): Uint8Array => Uint8Array.from(text, (ch) => ch.charCodeAt(0) & 0xff);
|
|
|
|
function pdfString(text: string): string {
|
|
return `(${text.replace(/\\/g, '\\\\').replace(/\(/g, '\\(').replace(/\)/g, '\\)')})`;
|
|
}
|
|
|
|
export interface PdfPage {
|
|
/** Lines of text, drawn top-down at 12pt with 14pt leading. `''` leaves a paragraph gap. */
|
|
lines?: string[];
|
|
/** Raw content stream, used instead of `lines`. */
|
|
raw?: string;
|
|
}
|
|
|
|
export interface PdfOptions {
|
|
title?: string;
|
|
/** Adds a standard security handler whose user password is not empty. */
|
|
encrypted?: boolean;
|
|
}
|
|
|
|
/** Builds a small but valid PDF (Helvetica, one content stream per page). */
|
|
export function buildPdf(pages: PdfPage[], options: PdfOptions = {}): Uint8Array {
|
|
const objects: string[] = [];
|
|
const add = (body: string): number => {
|
|
objects.push(body);
|
|
return objects.length;
|
|
};
|
|
const catalogId = add('');
|
|
const pagesId = add('');
|
|
const fontId = add('<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>');
|
|
|
|
const pageIds: number[] = [];
|
|
for (const page of pages) {
|
|
let stream = page.raw ?? '';
|
|
if (page.raw === undefined && page.lines && page.lines.length > 0) {
|
|
const ops: string[] = ['BT', '/F1 12 Tf', '14 TL', '72 720 Td'];
|
|
for (const line of page.lines) {
|
|
if (line === '') ops.push('0 -14 Td');
|
|
else ops.push(`${pdfString(line)} Tj`, 'T*');
|
|
}
|
|
ops.push('ET');
|
|
stream = ops.join('\n');
|
|
}
|
|
const contentId = add(`<< /Length ${stream.length} >>\nstream\n${stream}\nendstream`);
|
|
pageIds.push(
|
|
add(
|
|
`<< /Type /Page /Parent ${pagesId} 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 ${fontId} 0 R >> >> /Contents ${contentId} 0 R >>`,
|
|
),
|
|
);
|
|
}
|
|
objects[catalogId - 1] = `<< /Type /Catalog /Pages ${pagesId} 0 R >>`;
|
|
objects[pagesId - 1] = `<< /Type /Pages /Count ${pageIds.length} /Kids [${pageIds.map((id) => `${id} 0 R`).join(' ')}] >>`;
|
|
|
|
let infoId = 0;
|
|
if (options.title !== undefined && !options.encrypted) infoId = add(`<< /Title ${pdfString(options.title)} /Producer (study-desk tests) >>`);
|
|
let encryptId = 0;
|
|
if (options.encrypted) {
|
|
const o = 'a1'.repeat(32);
|
|
const u = 'b2'.repeat(32);
|
|
encryptId = add(`<< /Filter /Standard /V 1 /R 2 /Length 40 /O <${o}> /U <${u}> /P -44 >>`);
|
|
}
|
|
|
|
let out = '%PDF-1.4\n%\xE2\xE3\xCF\xD3\n';
|
|
const offsets: number[] = [];
|
|
objects.forEach((body, index) => {
|
|
offsets.push(out.length);
|
|
out += `${index + 1} 0 obj\n${body}\nendobj\n`;
|
|
});
|
|
const xrefOffset = out.length;
|
|
out += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
|
for (const offset of offsets) out += `${String(offset).padStart(10, '0')} 00000 n \n`;
|
|
out += `trailer\n<< /Size ${objects.length + 1} /Root ${catalogId} 0 R`;
|
|
if (infoId) out += ` /Info ${infoId} 0 R`;
|
|
if (encryptId) out += ` /Encrypt ${encryptId} 0 R /ID [<0123456789abcdef0123456789abcdef> <0123456789abcdef0123456789abcdef>]`;
|
|
out += ` >>\nstartxref\n${xrefOffset}\n%%EOF\n`;
|
|
return latin1(out);
|
|
}
|
|
|
|
const xmlEscape = (text: string): string => text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
|
|
const XML_HEADER = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>';
|
|
|
|
function coreProps(title: string | undefined): string {
|
|
return `${XML_HEADER}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">${
|
|
title === undefined ? '' : `<dc:title>${xmlEscape(title)}</dc:title>`
|
|
}<dc:creator>Tests</dc:creator></cp:coreProperties>`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DOCX
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export type DocxBlock =
|
|
| { type: 'heading'; level: number; text: string }
|
|
| { type: 'p'; text: string; bold?: string }
|
|
| { type: 'bullet'; text: string; level?: number }
|
|
| { type: 'number'; text: string; level?: number }
|
|
| { type: 'table'; rows: string[][] }
|
|
| { type: 'image' };
|
|
|
|
const W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"';
|
|
|
|
function docxRun(text: string, bold = false): string {
|
|
return `<w:r>${bold ? '<w:rPr><w:b/></w:rPr>' : ''}<w:t xml:space="preserve">${xmlEscape(text)}</w:t></w:r>`;
|
|
}
|
|
|
|
function docxBlock(block: DocxBlock): string {
|
|
switch (block.type) {
|
|
case 'heading':
|
|
return `<w:p><w:pPr><w:pStyle w:val="Heading${block.level}"/></w:pPr>${docxRun(block.text)}</w:p>`;
|
|
case 'p':
|
|
return `<w:p>${block.bold ? docxRun(block.bold, true) : ''}${docxRun(block.text)}</w:p>`;
|
|
case 'bullet':
|
|
case 'number':
|
|
return `<w:p><w:pPr><w:numPr><w:ilvl w:val="${block.level ?? 0}"/><w:numId w:val="${block.type === 'bullet' ? 1 : 2}"/></w:numPr></w:pPr>${docxRun(block.text)}</w:p>`;
|
|
case 'table':
|
|
return `<w:tbl>${block.rows
|
|
.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p>${docxRun(cell)}</w:p></w:tc>`).join('')}</w:tr>`)
|
|
.join('')}</w:tbl>`;
|
|
case 'image':
|
|
return '<w:p/>';
|
|
}
|
|
}
|
|
|
|
/** Builds a minimal DOCX with headings (style ids Heading1..6), lists, tables. */
|
|
export function buildDocx(blocks: DocxBlock[], options: { title?: string; extraFiles?: Record<string, Uint8Array> } = {}): Uint8Array {
|
|
const levels = (format: string) =>
|
|
[0, 1, 2].map((lvl) => `<w:lvl w:ilvl="${lvl}"><w:start w:val="1"/><w:numFmt w:val="${format}"/><w:lvlText w:val="%${lvl + 1}."/></w:lvl>`).join('');
|
|
const files: Record<string, Uint8Array> = {
|
|
'[Content_Types].xml': strToU8(
|
|
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Default Extension="png" ContentType="image/png"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/><Override PartName="/word/numbering.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml"/></Types>`,
|
|
),
|
|
'_rels/.rels': strToU8(
|
|
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties" Target="docProps/core.xml"/></Relationships>`,
|
|
),
|
|
'word/_rels/document.xml.rels': strToU8(
|
|
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering" Target="numbering.xml"/></Relationships>`,
|
|
),
|
|
'word/numbering.xml': strToU8(
|
|
`${XML_HEADER}<w:numbering ${W_NS}><w:abstractNum w:abstractNumId="0">${levels('bullet')}</w:abstractNum><w:abstractNum w:abstractNumId="1">${levels('decimal')}</w:abstractNum><w:num w:numId="1"><w:abstractNumId w:val="0"/></w:num><w:num w:numId="2"><w:abstractNumId w:val="1"/></w:num></w:numbering>`,
|
|
),
|
|
'word/document.xml': strToU8(`${XML_HEADER}<w:document ${W_NS}><w:body>${blocks.map(docxBlock).join('')}<w:sectPr/></w:body></w:document>`),
|
|
'docProps/core.xml': strToU8(coreProps(options.title)),
|
|
...options.extraFiles,
|
|
};
|
|
return zipSync(files);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// PPTX
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface PptxSlide {
|
|
/** File number: the slide is stored as ppt/slides/slide<file>.xml regardless of its position. */
|
|
file: number;
|
|
title?: string;
|
|
/** Body placeholder paragraphs. */
|
|
bullets?: Array<string | { text: string; level: number }>;
|
|
/** A free text box (not a placeholder). */
|
|
textBox?: string[];
|
|
table?: string[][];
|
|
notes?: string[];
|
|
}
|
|
|
|
const P_NS =
|
|
'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"';
|
|
|
|
function aParagraph(text: string, level = 0): string {
|
|
return `<a:p>${level ? `<a:pPr lvl="${level}"/>` : ''}<a:r><a:rPr lang="en-US"/><a:t>${xmlEscape(text)}</a:t></a:r></a:p>`;
|
|
}
|
|
|
|
function shape(id: number, placeholder: string | null, paragraphs: string): string {
|
|
const ph = placeholder === null ? '' : placeholder === 'body' ? '<p:ph idx="1"/>' : `<p:ph type="${placeholder}"/>`;
|
|
return `<p:sp><p:nvSpPr><p:cNvPr id="${id}" name="Shape ${id}"/><p:cNvSpPr/><p:nvPr>${ph}</p:nvPr></p:nvSpPr><p:spPr/><p:txBody><a:bodyPr/><a:lstStyle/>${paragraphs}</p:txBody></p:sp>`;
|
|
}
|
|
|
|
function slideXml(slide: PptxSlide): string {
|
|
const shapes: string[] = [];
|
|
let id = 2;
|
|
if (slide.title !== undefined) shapes.push(shape(id++, 'title', aParagraph(slide.title)));
|
|
if (slide.bullets) {
|
|
shapes.push(
|
|
shape(id++, 'body', slide.bullets.map((b) => (typeof b === 'string' ? aParagraph(b) : aParagraph(b.text, b.level))).join('')),
|
|
);
|
|
}
|
|
if (slide.textBox) shapes.push(shape(id++, null, slide.textBox.map((line) => aParagraph(line)).join('')));
|
|
if (slide.table) {
|
|
const rows = slide.table
|
|
.map((row) => `<a:tr h="370840">${row.map((cell) => `<a:tc><a:txBody><a:bodyPr/><a:lstStyle/>${aParagraph(cell)}</a:txBody><a:tcPr/></a:tc>`).join('')}</a:tr>`)
|
|
.join('');
|
|
shapes.push(
|
|
`<p:graphicFrame><p:nvGraphicFramePr><p:cNvPr id="${id++}" name="Table"/><p:cNvGraphicFramePr/><p:nvPr/></p:nvGraphicFramePr><p:xfrm/><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/table"><a:tbl><a:tblPr/><a:tblGrid/>${rows}</a:tbl></a:graphicData></a:graphic></p:graphicFrame>`,
|
|
);
|
|
}
|
|
// Every slide also carries a slide-number placeholder, which must be ignored.
|
|
shapes.push(shape(id++, 'sldNum', '<a:p><a:fld id="{1}" type="slidenum"><a:t>9</a:t></a:fld></a:p>'));
|
|
return `${XML_HEADER}<p:sld ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shapes.join('')}</p:spTree></p:cSld></p:sld>`;
|
|
}
|
|
|
|
function notesXml(lines: string[]): string {
|
|
return `${XML_HEADER}<p:notes ${P_NS}><p:cSld><p:spTree><p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr><p:grpSpPr/>${shape(
|
|
2,
|
|
'sldImg',
|
|
'',
|
|
)}${shape(3, 'body', lines.map((line) => aParagraph(line)).join(''))}${shape(4, 'sldNum', aParagraph('9'))}</p:spTree></p:cSld></p:notes>`;
|
|
}
|
|
|
|
/** Builds a minimal PPTX. Slides appear in the order given, whatever their file numbers. */
|
|
export function buildPptx(slides: PptxSlide[], options: { title?: string } = {}): Uint8Array {
|
|
const REL = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships';
|
|
const files: Record<string, Uint8Array> = {
|
|
'[Content_Types].xml': strToU8(
|
|
`${XML_HEADER}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>`,
|
|
),
|
|
'_rels/.rels': strToU8(
|
|
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/officeDocument" Target="ppt/presentation.xml"/></Relationships>`,
|
|
),
|
|
'docProps/core.xml': strToU8(coreProps(options.title)),
|
|
'ppt/presentation.xml': strToU8(
|
|
`${XML_HEADER}<p:presentation ${P_NS}><p:sldMasterIdLst/><p:sldIdLst>${slides
|
|
.map((slide, index) => `<p:sldId id="${256 + index}" r:id="rId${slide.file + 10}"/>`)
|
|
.join('')}</p:sldIdLst></p:presentation>`,
|
|
),
|
|
'ppt/_rels/presentation.xml.rels': strToU8(
|
|
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">${slides
|
|
.map((slide) => `<Relationship Id="rId${slide.file + 10}" Type="${REL}/slide" Target="slides/slide${slide.file}.xml"/>`)
|
|
.join('')}</Relationships>`,
|
|
),
|
|
};
|
|
for (const slide of slides) {
|
|
files[`ppt/slides/slide${slide.file}.xml`] = strToU8(slideXml(slide));
|
|
if (slide.notes) {
|
|
files[`ppt/slides/_rels/slide${slide.file}.xml.rels`] = strToU8(
|
|
`${XML_HEADER}<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="${REL}/notesSlide" Target="../notesSlides/notesSlide${slide.file}.xml"/></Relationships>`,
|
|
);
|
|
files[`ppt/notesSlides/notesSlide${slide.file}.xml`] = strToU8(notesXml(slide.notes));
|
|
}
|
|
}
|
|
return zipSync(files);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Misc
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** PNG signature plus an IHDR chunk header: enough for signature sniffing. */
|
|
export const TINY_PNG = Uint8Array.from([
|
|
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06,
|
|
0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89,
|
|
]);
|
|
|
|
export const TINY_JPEG = Uint8Array.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01]);
|
|
export const TINY_GIF = Uint8Array.from([...'GIF89a'].map((ch) => ch.charCodeAt(0)).concat([1, 0, 1, 0, 0, 0, 0]));
|
|
export const TINY_WEBP = Uint8Array.from([...'RIFF'].map((c) => c.charCodeAt(0)).concat([4, 0, 0, 0], [...'WEBP'].map((c) => c.charCodeAt(0))));
|
|
|
|
/** A stored zip whose central directory claims a tiny size for a highly compressible entry. */
|
|
export function buildLyingZip(name: string, realBytes: number): Uint8Array {
|
|
const zip = zipSync({ [name]: new Uint8Array(realBytes), 'word/document.xml': strToU8('<w:document/>') }, { level: 9 });
|
|
// Patch the uncompressed size of `name` in its central directory record to 16 bytes.
|
|
const view = new DataView(zip.buffer, zip.byteOffset, zip.byteLength);
|
|
for (let i = 0; i + 46 < zip.length; i++) {
|
|
if (view.getUint32(i, true) !== 0x02014b50) continue;
|
|
const nameLen = view.getUint16(i + 28, true);
|
|
const entryName = new TextDecoder().decode(zip.subarray(i + 46, i + 46 + nameLen));
|
|
if (entryName === name) view.setUint32(i + 24, 16, true);
|
|
}
|
|
return zip;
|
|
}
|