V1 remains intact on main and in history. This branch replaces it with a new application: node:sqlite storage with migrations, scrypt auth and sessions, content/SRS/attempt/stats services, document extraction and chunking, a provider-neutral AI layer (OpenAI, Anthropic, Gemini, OpenAI-compatible, Ollama), the generation pipeline and the tool-using assistant. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
53 lines
1.7 KiB
TypeScript
53 lines
1.7 KiB
TypeScript
/** Contract for the document extraction + chunking pipeline. */
|
|
|
|
export type DocKind = 'pdf' | 'docx' | 'pptx' | 'text' | 'markdown' | 'image';
|
|
|
|
export interface ExtractedSection {
|
|
/** Plain text (Markdown allowed) for this page / slide / section. */
|
|
text: string;
|
|
/** 1-based page (PDF) or slide (PPTX) number, when the format has one. */
|
|
page?: number;
|
|
/** Nearest heading or slide title, when known. */
|
|
heading?: string;
|
|
}
|
|
|
|
export interface ExtractResult {
|
|
kind: DocKind;
|
|
sections: ExtractedSection[];
|
|
/** Page or slide count for paged formats. */
|
|
pageCount?: number;
|
|
/** Non-fatal problems worth showing the user (e.g. "3 pages had no extractable text"). */
|
|
warnings: string[];
|
|
/** True when the file has little or no machine-readable text (scans, images) and needs AI vision/OCR. */
|
|
needsOcr: boolean;
|
|
/** Document title from metadata, when present. */
|
|
title?: string;
|
|
}
|
|
|
|
export interface DocChunk {
|
|
seq: number;
|
|
text: string;
|
|
heading: string;
|
|
pageStart: number | null;
|
|
pageEnd: number | null;
|
|
tokens: number;
|
|
}
|
|
|
|
export interface ChunkOptions {
|
|
/** Preferred chunk size in estimated tokens. */
|
|
targetTokens?: number;
|
|
/** Hard ceiling; a chunk is split before exceeding it. */
|
|
maxTokens?: number;
|
|
/** Tokens of trailing context repeated at the start of the next chunk. */
|
|
overlapTokens?: number;
|
|
}
|
|
|
|
/** Thrown for unrecoverable extraction problems; `message` is user-facing. */
|
|
export class ExtractionError extends Error {
|
|
code: 'unsupported' | 'encrypted' | 'corrupt' | 'empty' | 'too_large' | 'failed';
|
|
constructor(code: ExtractionError['code'], message: string, opts?: { cause?: unknown }) {
|
|
super(message, opts);
|
|
this.name = 'ExtractionError';
|
|
this.code = code;
|
|
}
|
|
}
|