Study/server/services/documents.ts
Elijah ccdad9728b
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 33s
Add drag reordering, exam deadlines, frozen shared quizzes, batched OCR and semantic search
- Drag-to-reorder for class materials, folders, quiz questions and deck cards. Pointer-based
  so touch works, plus keyboard (arrows/Home/End) with screen-reader announcements.
- Timed exams get a wall-clock deadline. The server refuses late answers, submits overdue
  exams with what was saved, and the countdown keeps running while the page is closed.
- Shared quiz sessions keep a frozen copy of their questions and flag later owner edits.
- Scanned PDFs are read in cached 8-page batches with real page numbers (pdf-lib), removing
  the 30 MB whole-file limit. Compatible providers now receive PDFs as file parts.
- Semantic search: embeddings for OpenAI, Gemini, Ollama and compatible servers, background
  indexing into chunk_vectors (migration 2), hybrid keyword + vector ranking for search, the
  assistant and focused generation, and a Settings -> AI section.

181 tests pass locally and in the Docker test stage on Node 24.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 11:48:20 -07:00

603 lines
24 KiB
TypeScript

/**
* Uploaded study materials: storage, the background processing queue
* (extract -> optional AI transcription -> chunk -> index) and retrieval.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { resolveModel, runChat } from '../ai/service.ts';
import { AIError, type ContentPart } from '../ai/types.ts';
import type { DB } from '../db.ts';
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
import { detectKind, extractDocument } from '../documents/extract.ts';
import { pdfPageCount, splitPdf } from '../documents/pdfSplit.ts';
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
import { HttpError, notFound, type AppContext } from '../http.ts';
import { ftsQuery, newId, now, parseJson } from '../util.ts';
import { getClass, resolveFolder } from './content.ts';
import { scheduleIndexing } from './semantic.ts';
export interface DocumentRow {
id: string;
class_id: string;
folder_id: string | null;
title: string;
filename: string;
mime: string;
kind: DocKind;
size: number;
sha256: string;
storage_name: string;
status: 'pending' | 'processing' | 'ready' | 'failed';
error: string;
progress: number;
page_count: number | null;
char_count: number;
token_count: number;
chunk_count: number;
needs_ocr: number;
warnings: string;
created_at: number;
updated_at: number;
}
export const mapDocument = (r: DocumentRow) => ({
id: r.id,
classId: r.class_id,
folderId: r.folder_id,
title: r.title,
filename: r.filename,
mime: r.mime,
docKind: r.kind,
size: r.size,
status: r.status,
error: r.error,
progress: r.progress,
pageCount: r.page_count,
charCount: r.char_count,
tokenCount: r.token_count,
chunkCount: r.chunk_count,
needsOcr: Boolean(r.needs_ocr),
warnings: parseJson<string[]>(r.warnings, []),
createdAt: r.created_at,
updatedAt: r.updated_at,
});
export type DocumentDto = ReturnType<typeof mapDocument>;
const MIME_BY_KIND: Record<DocKind, string> = {
pdf: 'application/pdf',
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
text: 'text/plain; charset=utf-8',
markdown: 'text/markdown; charset=utf-8',
image: 'application/octet-stream',
};
function imageMime(bytes: Uint8Array): string {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png';
if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif';
if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp';
return 'application/octet-stream';
}
export function getDocument(db: DB, id: string): DocumentRow {
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) throw notFound('Document');
return row;
}
export function documentPath(app: AppContext, row: Pick<DocumentRow, 'storage_name'>): string {
// basename() guarantees a stored name can never escape the uploads folder.
return path.join(app.config.uploadsDir, path.basename(row.storage_name));
}
function titleFromFilename(filename: string): string {
const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim();
return (base || 'Untitled document').slice(0, 200);
}
/* ------------------------------ ingest ------------------------------ */
export function ingestUpload(
app: AppContext,
input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string },
): DocumentRow {
const { db } = app;
getClass(db, input.classId);
if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`);
if (input.bytes.length > app.config.maxUploadBytes) {
throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
}
const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload';
let kind = detectKind(filename, input.mime, input.bytes);
// A file whose name claims a binary format but whose bytes say otherwise (a renamed
// executable, say) is refused here instead of being stored and failing later.
if (kind === 'pdf' && !Buffer.from(input.bytes.subarray(0, 1024)).includes('%PDF')) kind = null;
if ((kind === 'docx' || kind === 'pptx') && !(input.bytes[0] === 0x50 && input.bytes[1] === 0x4b) && !(input.bytes[0] === 0xd0 && input.bytes[1] === 0xcf)) kind = null;
if (!kind) {
throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`);
}
const id = newId();
const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind];
const storageName = `${id}${ext}`;
fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes);
const at = now();
const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1;
db.run(
`INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`,
id,
input.classId,
resolveFolder(db, input.classId, input.folderId),
(input.title?.trim() || titleFromFilename(filename)).slice(0, 200),
filename,
kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind],
kind,
input.bytes.length,
crypto.createHash('sha256').update(input.bytes).digest('hex'),
storageName,
sort,
at,
at,
);
enqueue(app, id);
return getDocument(db, id);
}
export function deleteDocument(app: AppContext, id: string): void {
const row = getDocument(app.db, id);
app.db.tx(() => {
app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id);
app.db.run('DELETE FROM documents WHERE id = ?', id);
});
fs.rmSync(documentPath(app, row), { force: true });
fs.rmSync(ocrCachePath(app, row), { force: true });
}
/* ------------------------------ processing queue ------------------------------ */
const queues = new WeakMap<DB, { pending: string[]; running: boolean; ocr: Set<string>; idle: Promise<void>; wake: () => void }>();
function queueFor(app: AppContext) {
let q = queues.get(app.db);
if (!q) {
q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined };
queues.set(app.db, q);
}
return q;
}
export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void {
const q = queueFor(app);
if (opts.ocr) q.ocr.add(id);
if (!q.pending.includes(id)) q.pending.push(id);
if (!q.running) {
q.running = true;
q.idle = (async () => {
// Yield first so the HTTP response for the upload is sent before heavy parsing starts.
await new Promise((r) => setTimeout(r, 10));
while (q.pending.length > 0) {
const next = q.pending.shift()!;
const forceOcr = q.ocr.delete(next);
try {
await processDocument(app, next, { forceOcr });
} catch (e) {
app.log('error', `Document ${next} failed unexpectedly`, e);
}
}
q.running = false;
})();
}
}
/** Resolves when the queue is empty. Used by tests and graceful shutdown. */
export function queueIdle(app: AppContext): Promise<void> {
return queueFor(app).idle;
}
/** Re-queues anything a restart interrupted. */
export function resumeQueue(app: AppContext): number {
const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`);
for (const r of rows) enqueue(app, r.id);
return rows.length;
}
const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
- Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information.
- Keep headings, lists and tables (as Markdown tables).
- For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown.
- For a multi-page document, start each page with a line of the form "## Page N".
- If something is illegible, write [illegible].
Return only the transcription.`;
/** Long scanned PDFs are read this many pages at a time. */
export const OCR_PAGES_PER_BATCH = 8;
/** Largest single request sent to a model (providers cap inline files around 32 MB). */
const OCR_MAX_BATCH_BYTES = 24 * 1024 * 1024;
/** Upper bound on pages read with AI for one document, to keep costs predictable. */
export const OCR_MAX_PAGES = 600;
interface OcrCache {
modelId: string;
/** Finished batches keyed by "start-end", so an interrupted read resumes instead of paying twice. */
batches: Record<string, ExtractedSection[]>;
}
const ocrCachePath = (app: AppContext, row: Pick<DocumentRow, 'storage_name'>) => `${documentPath(app, row)}.ocr.json`;
/** Splits a "## Page N" transcription into page sections; text without markers goes to `fallbackPage`. */
export function splitTranscription(text: string, fallbackPage: number | undefined, range?: { start: number; end: number }): ExtractedSection[] {
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
if (pages.length < 3) return text.trim() ? [{ text: text.trim(), page: fallbackPage }] : [];
const sections: ExtractedSection[] = [];
if (pages[0].trim()) sections.push({ text: pages[0].trim(), page: fallbackPage });
for (let i = 1; i < pages.length; i += 2) {
const body = (pages[i + 1] ?? '').trim();
let page = Number(pages[i]);
// Models sometimes number a batch from 1; map those back onto the batch's real pages.
if (range && (page < range.start || page > range.end) && page >= 1 && page <= range.end - range.start + 1) page = range.start + page - 1;
if (body) sections.push({ text: body, page });
}
return sections;
}
interface Transcription {
sections: ExtractedSection[];
warnings: string[];
}
/**
* Uses a vision-capable model to read an image or a scanned PDF. PDFs are sent in batches of a
* few pages, with finished batches cached beside the upload so a restart or a failed batch does
* not repeat paid work. Returns null when no vision model is configured.
*/
async function transcribeWithVision(
app: AppContext,
row: DocumentRow,
bytes: Uint8Array,
opts: { fresh: boolean; onProgress?: (fraction: number) => void },
): Promise<Transcription | null> {
let vision;
try {
vision = resolveModel(app, 'vision');
} catch (err) {
if (err instanceof AIError && err.code === 'not_configured') return null;
throw err;
}
if (!vision.model.capabilities.vision) return null;
const ask = async (part: ContentPart, prompt: string) => {
const result = await runChat(app, {
task: 'vision',
purpose: 'document transcription',
messages: [{ role: 'user', content: [part, { type: 'text', text: prompt }] }],
maxOutputTokens: 32000,
reasoningEffort: 'low',
});
return result.text.trim();
};
if (row.kind === 'image') {
const text = await ask({ type: 'image', mime: row.mime, data: Buffer.from(bytes).toString('base64') }, TRANSCRIBE_PROMPT);
return { sections: text ? [{ text }] : [], warnings: [] };
}
if (!vision.model.capabilities.pdf) {
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
}
const cacheFile = ocrCachePath(app, row);
let cache: OcrCache = { modelId: vision.model.id, batches: {} };
if (opts.fresh) fs.rmSync(cacheFile, { force: true });
else {
try {
const saved = JSON.parse(fs.readFileSync(cacheFile, 'utf8')) as OcrCache;
if (saved.modelId === vision.model.id && saved.batches) cache = saved;
} catch {
/* no cache yet */
}
}
const save = () => fs.writeFileSync(cacheFile, JSON.stringify(cache));
const warnings: string[] = [];
let total: number;
try {
total = await pdfPageCount(bytes);
} catch {
throw new AIError('bad_request', 'This PDF could not be split into pages for reading. It may be damaged or encrypted.');
}
const limit = Math.min(total, OCR_MAX_PAGES);
if (total > OCR_MAX_PAGES) warnings.push(`Only the first ${OCR_MAX_PAGES} of ${total} pages were read with AI. Split the file to read the rest.`);
const sections: ExtractedSection[] = [];
for await (const batch of splitPdf(bytes, OCR_PAGES_PER_BATCH, OCR_MAX_BATCH_BYTES)) {
if (batch.start > limit) break;
const key = `${batch.start}-${batch.end}`;
const label = batch.start === batch.end ? `Page ${batch.start}` : `Pages ${batch.start}-${batch.end}`;
let got: ExtractedSection[] | undefined = cache.batches[key];
if (!got) {
if (batch.bytes.length > OCR_MAX_BATCH_BYTES) {
warnings.push(`${label} is too large to send to an AI model and was skipped.`);
} else {
const prompt = `${TRANSCRIBE_PROMPT}\n\nThis file contains ${label.toLowerCase()} of a ${total}-page document. Number the "## Page N" headings with those real page numbers (${batch.start} to ${batch.end}).`;
const part: ContentPart = { type: 'file', mime: 'application/pdf', data: Buffer.from(batch.bytes).toString('base64'), filename: `${row.filename.replace(/\.pdf$/i, '')}-p${key}.pdf` };
for (let attempt = 1; attempt <= 2 && !got; attempt++) {
try {
got = splitTranscription(await ask(part, prompt), batch.start, batch);
} catch (err) {
// Configuration problems will not fix themselves on the next batch.
if (err instanceof AIError && ['not_configured', 'unsupported', 'quota'].includes(err.code)) throw err;
if (attempt === 2) warnings.push(`${label} could not be read: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
if (got) {
cache.batches[key] = got;
save();
}
}
}
if (got) sections.push(...got);
opts.onProgress?.(Math.min(limit, batch.end) / limit);
}
return { sections, warnings };
}
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
const { db } = app;
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
if (!row) return;
const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id);
db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id);
try {
const file = documentPath(app, row);
if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.');
const bytes = new Uint8Array(await fs.promises.readFile(file));
let lastWrite = 0;
const result = await extractDocument({
bytes,
filename: row.filename,
mime: row.mime,
onProgress: (done, total) => {
const t = Date.now();
if (t - lastWrite > 400 && total > 0) {
lastWrite = t;
setProgress((done / total) * 0.8);
}
},
});
const warnings = [...result.warnings];
let sections = result.sections;
let needsOcr = result.needsOcr;
if (needsOcr || opts.forceOcr) {
setProgress(0.82);
if (row.kind === 'image' || row.kind === 'pdf') {
try {
const transcribed = await transcribeWithVision(app, row, bytes, { fresh: Boolean(opts.forceOcr), onProgress: (f) => setProgress(0.82 + f * 0.16) });
if (transcribed) warnings.push(...transcribed.warnings);
if (transcribed === null) {
warnings.push(
row.kind === 'image'
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
);
} else if (transcribed.sections.length > 0) {
sections = transcribed.sections;
needsOcr = false;
} else {
warnings.push('The AI model did not find any readable text in this file.');
}
} catch (err) {
warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`);
}
}
}
const chunks = chunkSections(sections);
const chars = sections.reduce((n, s) => n + s.text.length, 0);
const tokens = chunks.reduce((n, c) => n + c.tokens, 0);
db.tx(() => {
db.run('DELETE FROM chunks WHERE document_id = ?', id);
const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id);
if (!current) return; // deleted while processing
for (const c of chunks) {
db.run(
'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
id,
current.class_id,
c.seq,
c.text,
c.heading,
c.pageStart,
c.pageEnd,
c.tokens,
);
}
db.run(
`UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`,
result.pageCount ?? null,
chars,
tokens,
chunks.length,
needsOcr ? 1 : 0,
JSON.stringify([...new Set(warnings)].slice(0, 12)),
now(),
id,
);
});
scheduleIndexing(app);
} catch (err) {
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id);
}
}
/* ------------------------------ retrieval ------------------------------ */
export interface ChunkHit {
id: number;
documentId: string;
documentTitle: string;
seq: number;
text: string;
heading: string;
pageStart: number | null;
pageEnd: number | null;
tokens: number;
score?: number;
}
type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number };
const mapHit = (r: ChunkJoin): ChunkHit => ({
id: r.id,
documentId: r.document_id,
documentTitle: r.doc_title,
seq: r.seq,
text: r.text,
heading: r.heading,
pageStart: r.page_start,
pageEnd: r.page_end,
tokens: r.tokens,
score: r.score,
});
export interface ChunkScope {
classId?: string | null;
documentIds?: string[];
}
function scopeSql(scope: ChunkScope): { sql: string; params: string[] } {
if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds };
if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] };
return { sql: '', params: [] };
}
/** BM25 full-text search over document chunks. */
export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] {
const match = ftsQuery(query, 'or');
if (!match) return [];
const { sql, params } = scopeSql(scope);
try {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id
WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`,
match,
...params,
limit,
)
.map(mapHit);
} catch {
return [];
}
}
export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] {
return db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`,
documentId,
offset,
limit,
)
.map(mapHit);
}
export function chunkById(db: DB, id: number): ChunkHit | null {
const row = db.get<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`,
id,
);
return row ? mapHit(row) : null;
}
export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' | 'pageEnd'>): string {
if (hit.pageStart === null) return hit.documentTitle;
return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`;
}
/**
* Chooses source material for a generation job and splits it into `groups` batches that
* each fit `budgetTokens`.
* - With a focus, the best-matching chunks are used (kept in document order).
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
* so every part of the document is represented.
*/
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number; ranked?: ChunkHit[] }): ChunkHit[][] {
const { sql, params } = scopeSql(scope);
if (!sql) return [];
const all = db
.all<ChunkJoin>(
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`,
...params,
)
.map(mapHit);
if (all.length === 0) return [];
const capacity = opts.budgetTokens * opts.groups;
const total = all.reduce((n, c) => n + c.tokens, 0);
let chosen = all;
if (opts.focus?.trim()) {
// Callers pass a hybrid (keyword + semantic) ranking when they have one.
const ranked = opts.ranked ?? searchChunks(db, opts.focus, scope, 400);
if (ranked.length >= 3) {
const keep = new Set<number>();
let used = 0;
for (const hit of ranked) {
if (used + hit.tokens > capacity) break;
keep.add(hit.id);
used += hit.tokens;
}
chosen = all.filter((c) => keep.has(c.id));
}
}
if (chosen === all && total > capacity) {
// Even sampling: walk the document and keep chunks at a steady stride.
const stride = total / capacity;
const picked: ChunkHit[] = [];
let acc = 0;
let next = 0;
for (const c of all) {
if (acc >= next) {
picked.push(c);
next += c.tokens * stride;
}
acc += c.tokens;
}
chosen = picked;
}
// Contiguous groups of roughly equal size, none above the budget.
const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0);
const groupCount = Math.max(1, Math.min(opts.groups, chosen.length));
const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount));
const groups: ChunkHit[][] = [];
let current: ChunkHit[] = [];
let size = 0;
for (const c of chosen) {
if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) {
groups.push(current);
current = [];
size = 0;
}
if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full
current.push(c);
size += c.tokens;
}
if (current.length > 0) groups.push(current);
return groups;
}
export function formatExcerpts(chunks: ChunkHit[]): string {
return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n');
}
export { estimateTokens };