504 lines
20 KiB
TypeScript
504 lines
20 KiB
TypeScript
/**
|
|
* Uploaded study materials: storage, the background processing queue
|
|
* (extract -> optional AI transcription -> chunk -> index) and retrieval.
|
|
*/
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import crypto from 'node:crypto';
|
|
import { resolveModel, runChat } from '../ai/service.ts';
|
|
import { AIError, type ContentPart } from '../ai/types.ts';
|
|
import type { DB } from '../db.ts';
|
|
import { chunkSections, estimateTokens } from '../documents/chunk.ts';
|
|
import { detectKind, extractDocument } from '../documents/extract.ts';
|
|
import { ExtractionError, type DocKind, type ExtractedSection } from '../documents/types.ts';
|
|
import { HttpError, notFound, type AppContext } from '../http.ts';
|
|
import { ftsQuery, newId, now, parseJson } from '../util.ts';
|
|
import { getClass, resolveFolder } from './content.ts';
|
|
|
|
export interface DocumentRow {
|
|
id: string;
|
|
class_id: string;
|
|
folder_id: string | null;
|
|
title: string;
|
|
filename: string;
|
|
mime: string;
|
|
kind: DocKind;
|
|
size: number;
|
|
sha256: string;
|
|
storage_name: string;
|
|
status: 'pending' | 'processing' | 'ready' | 'failed';
|
|
error: string;
|
|
progress: number;
|
|
page_count: number | null;
|
|
char_count: number;
|
|
token_count: number;
|
|
chunk_count: number;
|
|
needs_ocr: number;
|
|
warnings: string;
|
|
created_at: number;
|
|
updated_at: number;
|
|
}
|
|
|
|
export const mapDocument = (r: DocumentRow) => ({
|
|
id: r.id,
|
|
classId: r.class_id,
|
|
folderId: r.folder_id,
|
|
title: r.title,
|
|
filename: r.filename,
|
|
mime: r.mime,
|
|
docKind: r.kind,
|
|
size: r.size,
|
|
status: r.status,
|
|
error: r.error,
|
|
progress: r.progress,
|
|
pageCount: r.page_count,
|
|
charCount: r.char_count,
|
|
tokenCount: r.token_count,
|
|
chunkCount: r.chunk_count,
|
|
needsOcr: Boolean(r.needs_ocr),
|
|
warnings: parseJson<string[]>(r.warnings, []),
|
|
createdAt: r.created_at,
|
|
updatedAt: r.updated_at,
|
|
});
|
|
export type DocumentDto = ReturnType<typeof mapDocument>;
|
|
|
|
const MIME_BY_KIND: Record<DocKind, string> = {
|
|
pdf: 'application/pdf',
|
|
docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
text: 'text/plain; charset=utf-8',
|
|
markdown: 'text/markdown; charset=utf-8',
|
|
image: 'application/octet-stream',
|
|
};
|
|
|
|
function imageMime(bytes: Uint8Array): string {
|
|
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'image/png';
|
|
if (bytes[0] === 0xff && bytes[1] === 0xd8) return 'image/jpeg';
|
|
if (bytes[0] === 0x47 && bytes[1] === 0x49) return 'image/gif';
|
|
if (bytes[0] === 0x52 && bytes[8] === 0x57) return 'image/webp';
|
|
return 'application/octet-stream';
|
|
}
|
|
|
|
export function getDocument(db: DB, id: string): DocumentRow {
|
|
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
|
|
if (!row) throw notFound('Document');
|
|
return row;
|
|
}
|
|
|
|
export function documentPath(app: AppContext, row: Pick<DocumentRow, 'storage_name'>): string {
|
|
// basename() guarantees a stored name can never escape the uploads folder.
|
|
return path.join(app.config.uploadsDir, path.basename(row.storage_name));
|
|
}
|
|
|
|
function titleFromFilename(filename: string): string {
|
|
const base = filename.replace(/\.[^.]+$/, '').replace(/[_]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
return (base || 'Untitled document').slice(0, 200);
|
|
}
|
|
|
|
/* ------------------------------ ingest ------------------------------ */
|
|
|
|
export function ingestUpload(
|
|
app: AppContext,
|
|
input: { classId: string; folderId?: string | null; filename: string; mime?: string; bytes: Uint8Array; title?: string },
|
|
): DocumentRow {
|
|
const { db } = app;
|
|
getClass(db, input.classId);
|
|
if (input.bytes.length === 0) throw new HttpError(422, 'empty_file', `"${input.filename}" is empty.`);
|
|
if (input.bytes.length > app.config.maxUploadBytes) {
|
|
throw new HttpError(413, 'too_large', `"${input.filename}" is larger than the ${Math.round(app.config.maxUploadBytes / 1048576)} MB upload limit.`);
|
|
}
|
|
const filename = path.basename(input.filename.replace(/\\/g, '/')).slice(0, 200) || 'upload';
|
|
let kind = detectKind(filename, input.mime, input.bytes);
|
|
// A file whose name claims a binary format but whose bytes say otherwise (a renamed
|
|
// executable, say) is refused here instead of being stored and failing later.
|
|
if (kind === 'pdf' && !Buffer.from(input.bytes.subarray(0, 1024)).includes('%PDF')) kind = null;
|
|
if ((kind === 'docx' || kind === 'pptx') && !(input.bytes[0] === 0x50 && input.bytes[1] === 0x4b) && !(input.bytes[0] === 0xd0 && input.bytes[1] === 0xcf)) kind = null;
|
|
if (!kind) {
|
|
throw new HttpError(415, 'unsupported_type', `"${filename}" is not a supported file type. Upload PDF, Word (.docx), PowerPoint (.pptx), text, Markdown, or an image.`);
|
|
}
|
|
const id = newId();
|
|
const ext = { pdf: '.pdf', docx: '.docx', pptx: '.pptx', text: '.txt', markdown: '.md', image: path.extname(filename).toLowerCase().replace(/[^.a-z0-9]/g, '') || '.img' }[kind];
|
|
const storageName = `${id}${ext}`;
|
|
fs.writeFileSync(path.join(app.config.uploadsDir, storageName), input.bytes);
|
|
const at = now();
|
|
const sort = (db.get<{ m: number | null }>('SELECT MAX(sort_order) AS m FROM documents WHERE class_id = ?', input.classId)?.m ?? -1) + 1;
|
|
db.run(
|
|
`INSERT INTO documents (id, class_id, folder_id, title, filename, mime, kind, size, sha256, storage_name, status, sort_order, created_at, updated_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?, ?)`,
|
|
id,
|
|
input.classId,
|
|
resolveFolder(db, input.classId, input.folderId),
|
|
(input.title?.trim() || titleFromFilename(filename)).slice(0, 200),
|
|
filename,
|
|
kind === 'image' ? imageMime(input.bytes) : MIME_BY_KIND[kind],
|
|
kind,
|
|
input.bytes.length,
|
|
crypto.createHash('sha256').update(input.bytes).digest('hex'),
|
|
storageName,
|
|
sort,
|
|
at,
|
|
at,
|
|
);
|
|
enqueue(app, id);
|
|
return getDocument(db, id);
|
|
}
|
|
|
|
export function deleteDocument(app: AppContext, id: string): void {
|
|
const row = getDocument(app.db, id);
|
|
app.db.tx(() => {
|
|
app.db.run(`DELETE FROM shares WHERE kind = 'document' AND target_id = ?`, id);
|
|
app.db.run('DELETE FROM documents WHERE id = ?', id);
|
|
});
|
|
fs.rmSync(documentPath(app, row), { force: true });
|
|
}
|
|
|
|
/* ------------------------------ processing queue ------------------------------ */
|
|
|
|
const queues = new WeakMap<DB, { pending: string[]; running: boolean; ocr: Set<string>; idle: Promise<void>; wake: () => void }>();
|
|
|
|
function queueFor(app: AppContext) {
|
|
let q = queues.get(app.db);
|
|
if (!q) {
|
|
q = { pending: [], running: false, ocr: new Set(), idle: Promise.resolve(), wake: () => undefined };
|
|
queues.set(app.db, q);
|
|
}
|
|
return q;
|
|
}
|
|
|
|
export function enqueue(app: AppContext, id: string, opts: { ocr?: boolean } = {}): void {
|
|
const q = queueFor(app);
|
|
if (opts.ocr) q.ocr.add(id);
|
|
if (!q.pending.includes(id)) q.pending.push(id);
|
|
if (!q.running) {
|
|
q.running = true;
|
|
q.idle = (async () => {
|
|
// Yield first so the HTTP response for the upload is sent before heavy parsing starts.
|
|
await new Promise((r) => setTimeout(r, 10));
|
|
while (q.pending.length > 0) {
|
|
const next = q.pending.shift()!;
|
|
const forceOcr = q.ocr.delete(next);
|
|
try {
|
|
await processDocument(app, next, { forceOcr });
|
|
} catch (e) {
|
|
app.log('error', `Document ${next} failed unexpectedly`, e);
|
|
}
|
|
}
|
|
q.running = false;
|
|
})();
|
|
}
|
|
}
|
|
|
|
/** Resolves when the queue is empty. Used by tests and graceful shutdown. */
|
|
export function queueIdle(app: AppContext): Promise<void> {
|
|
return queueFor(app).idle;
|
|
}
|
|
|
|
/** Re-queues anything a restart interrupted. */
|
|
export function resumeQueue(app: AppContext): number {
|
|
const rows = app.db.all<{ id: string }>(`SELECT id FROM documents WHERE status IN ('pending', 'processing') ORDER BY created_at`);
|
|
for (const r of rows) enqueue(app, r.id);
|
|
return rows.length;
|
|
}
|
|
|
|
const TRANSCRIBE_PROMPT = `Transcribe this study material into clean Markdown.
|
|
- Reproduce all readable text faithfully, in reading order. Do not summarise and do not add information.
|
|
- Keep headings, lists and tables (as Markdown tables).
|
|
- For diagrams, charts and labelled figures, add a short description in italics that includes every label and value shown.
|
|
- For a multi-page document, start each page with a line of the form "## Page N".
|
|
- If something is illegible, write [illegible].
|
|
Return only the transcription.`;
|
|
|
|
/** Uses a vision-capable model to read an image or a scanned PDF. Returns null when no such model is configured. */
|
|
async function transcribeWithVision(app: AppContext, row: DocumentRow, bytes: Uint8Array): Promise<ExtractedSection[] | null> {
|
|
const data = Buffer.from(bytes).toString('base64');
|
|
const part: ContentPart = row.kind === 'image' ? { type: 'image', mime: row.mime, data } : { type: 'file', mime: 'application/pdf', data, filename: row.filename };
|
|
if (row.kind === 'pdf' && bytes.length > 30 * 1024 * 1024) {
|
|
throw new AIError('bad_request', 'This scanned PDF is too large to send to an AI model for reading (30 MB limit). Split it into smaller files.');
|
|
}
|
|
let result;
|
|
try {
|
|
const vision = resolveModel(app, 'vision');
|
|
if (!vision.model.capabilities.vision) return null;
|
|
if (row.kind === 'pdf' && !vision.model.capabilities.pdf) {
|
|
throw new AIError('unsupported', `${vision.model.label} cannot read PDF files directly. Choose a model with PDF support for "Vision and OCR" in Settings.`);
|
|
}
|
|
result = await runChat(app, {
|
|
task: 'vision',
|
|
purpose: 'document transcription',
|
|
messages: [{ role: 'user', content: [part, { type: 'text', text: TRANSCRIBE_PROMPT }] }],
|
|
maxOutputTokens: 32000,
|
|
reasoningEffort: 'low',
|
|
});
|
|
} catch (err) {
|
|
if (err instanceof AIError && err.code === 'not_configured') return null;
|
|
throw err;
|
|
}
|
|
const text = result.text.trim();
|
|
if (!text) return [];
|
|
// Split on the "## Page N" markers the prompt asks for so citations keep page numbers.
|
|
const pages = text.split(/^##\s*Page\s+(\d+)\s*$/im);
|
|
if (pages.length < 3) return [{ text, page: row.kind === 'image' ? undefined : 1 }];
|
|
const sections: ExtractedSection[] = [];
|
|
if (pages[0].trim()) sections.push({ text: pages[0].trim() });
|
|
for (let i = 1; i < pages.length; i += 2) {
|
|
const body = (pages[i + 1] ?? '').trim();
|
|
if (body) sections.push({ text: body, page: Number(pages[i]) });
|
|
}
|
|
return sections;
|
|
}
|
|
|
|
export async function processDocument(app: AppContext, id: string, opts: { forceOcr?: boolean } = {}): Promise<void> {
|
|
const { db } = app;
|
|
const row = db.get<DocumentRow>('SELECT * FROM documents WHERE id = ?', id);
|
|
if (!row) return;
|
|
const setProgress = (p: number) => db.run('UPDATE documents SET progress = ?, updated_at = ? WHERE id = ?', Math.round(p * 100) / 100, now(), id);
|
|
db.run(`UPDATE documents SET status = 'processing', error = '', progress = 0, updated_at = ? WHERE id = ?`, now(), id);
|
|
|
|
try {
|
|
const file = documentPath(app, row);
|
|
if (!fs.existsSync(file)) throw new ExtractionError('failed', 'The uploaded file is missing from storage. Upload it again.');
|
|
const bytes = new Uint8Array(await fs.promises.readFile(file));
|
|
let lastWrite = 0;
|
|
const result = await extractDocument({
|
|
bytes,
|
|
filename: row.filename,
|
|
mime: row.mime,
|
|
onProgress: (done, total) => {
|
|
const t = Date.now();
|
|
if (t - lastWrite > 400 && total > 0) {
|
|
lastWrite = t;
|
|
setProgress((done / total) * 0.8);
|
|
}
|
|
},
|
|
});
|
|
const warnings = [...result.warnings];
|
|
let sections = result.sections;
|
|
let needsOcr = result.needsOcr;
|
|
|
|
if (needsOcr || opts.forceOcr) {
|
|
setProgress(0.82);
|
|
if (row.kind === 'image' || row.kind === 'pdf') {
|
|
try {
|
|
const transcribed = await transcribeWithVision(app, row, bytes);
|
|
if (transcribed === null) {
|
|
warnings.push(
|
|
row.kind === 'image'
|
|
? 'Images need an AI model with vision to be read. Add one in Settings, then choose "Read with AI".'
|
|
: 'This PDF has no selectable text (it looks scanned). Add an AI model with vision in Settings, then choose "Read with AI".',
|
|
);
|
|
} else if (transcribed.length > 0) {
|
|
sections = transcribed;
|
|
needsOcr = false;
|
|
} else {
|
|
warnings.push('The AI model did not find any readable text in this file.');
|
|
}
|
|
} catch (err) {
|
|
warnings.push(`Reading with AI failed: ${err instanceof Error ? err.message : 'unknown error'}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
const chunks = chunkSections(sections);
|
|
const chars = sections.reduce((n, s) => n + s.text.length, 0);
|
|
const tokens = chunks.reduce((n, c) => n + c.tokens, 0);
|
|
db.tx(() => {
|
|
db.run('DELETE FROM chunks WHERE document_id = ?', id);
|
|
const current = db.get<{ class_id: string }>('SELECT class_id FROM documents WHERE id = ?', id);
|
|
if (!current) return; // deleted while processing
|
|
for (const c of chunks) {
|
|
db.run(
|
|
'INSERT INTO chunks (document_id, class_id, seq, text, heading, page_start, page_end, tokens) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
|
|
id,
|
|
current.class_id,
|
|
c.seq,
|
|
c.text,
|
|
c.heading,
|
|
c.pageStart,
|
|
c.pageEnd,
|
|
c.tokens,
|
|
);
|
|
}
|
|
db.run(
|
|
`UPDATE documents SET status = 'ready', error = '', progress = 1, page_count = ?, char_count = ?, token_count = ?, chunk_count = ?, needs_ocr = ?, warnings = ?, updated_at = ? WHERE id = ?`,
|
|
result.pageCount ?? null,
|
|
chars,
|
|
tokens,
|
|
chunks.length,
|
|
needsOcr ? 1 : 0,
|
|
JSON.stringify([...new Set(warnings)].slice(0, 8)),
|
|
now(),
|
|
id,
|
|
);
|
|
});
|
|
} catch (err) {
|
|
const message = err instanceof ExtractionError ? err.message : 'Something went wrong while reading this file. Try uploading it again.';
|
|
if (!(err instanceof ExtractionError)) app.log('error', `Document ${id} processing error`, err);
|
|
db.run(`UPDATE documents SET status = 'failed', error = ?, progress = 0, updated_at = ? WHERE id = ?`, message, now(), id);
|
|
}
|
|
}
|
|
|
|
/* ------------------------------ retrieval ------------------------------ */
|
|
|
|
export interface ChunkHit {
|
|
id: number;
|
|
documentId: string;
|
|
documentTitle: string;
|
|
seq: number;
|
|
text: string;
|
|
heading: string;
|
|
pageStart: number | null;
|
|
pageEnd: number | null;
|
|
tokens: number;
|
|
score?: number;
|
|
}
|
|
|
|
type ChunkJoin = { id: number; document_id: string; doc_title: string; seq: number; text: string; heading: string; page_start: number | null; page_end: number | null; tokens: number; score?: number };
|
|
|
|
const mapHit = (r: ChunkJoin): ChunkHit => ({
|
|
id: r.id,
|
|
documentId: r.document_id,
|
|
documentTitle: r.doc_title,
|
|
seq: r.seq,
|
|
text: r.text,
|
|
heading: r.heading,
|
|
pageStart: r.page_start,
|
|
pageEnd: r.page_end,
|
|
tokens: r.tokens,
|
|
score: r.score,
|
|
});
|
|
|
|
export interface ChunkScope {
|
|
classId?: string | null;
|
|
documentIds?: string[];
|
|
}
|
|
|
|
function scopeSql(scope: ChunkScope): { sql: string; params: string[] } {
|
|
if (scope.documentIds?.length) return { sql: `AND ch.document_id IN (${scope.documentIds.map(() => '?').join(',')})`, params: scope.documentIds };
|
|
if (scope.classId) return { sql: 'AND ch.class_id = ?', params: [scope.classId] };
|
|
return { sql: '', params: [] };
|
|
}
|
|
|
|
/** BM25 full-text search over document chunks. */
|
|
export function searchChunks(db: DB, query: string, scope: ChunkScope, limit = 8): ChunkHit[] {
|
|
const match = ftsQuery(query, 'or');
|
|
if (!match) return [];
|
|
const { sql, params } = scopeSql(scope);
|
|
try {
|
|
return db
|
|
.all<ChunkJoin>(
|
|
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens, bm25(chunks_fts, 1.0, 2.0) AS score
|
|
FROM chunks_fts JOIN chunks ch ON ch.id = chunks_fts.rowid JOIN documents d ON d.id = ch.document_id
|
|
WHERE chunks_fts MATCH ? ${sql} ORDER BY score LIMIT ?`,
|
|
match,
|
|
...params,
|
|
limit,
|
|
)
|
|
.map(mapHit);
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
export function listChunks(db: DB, documentId: string, offset = 0, limit = 50): ChunkHit[] {
|
|
return db
|
|
.all<ChunkJoin>(
|
|
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
|
|
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.document_id = ? AND ch.seq >= ? ORDER BY ch.seq LIMIT ?`,
|
|
documentId,
|
|
offset,
|
|
limit,
|
|
)
|
|
.map(mapHit);
|
|
}
|
|
|
|
export function chunkById(db: DB, id: number): ChunkHit | null {
|
|
const row = db.get<ChunkJoin>(
|
|
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
|
|
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE ch.id = ?`,
|
|
id,
|
|
);
|
|
return row ? mapHit(row) : null;
|
|
}
|
|
|
|
export function sourceLabel(hit: Pick<ChunkHit, 'documentTitle' | 'pageStart' | 'pageEnd'>): string {
|
|
if (hit.pageStart === null) return hit.documentTitle;
|
|
return hit.pageEnd && hit.pageEnd !== hit.pageStart ? `${hit.documentTitle}, pp. ${hit.pageStart}-${hit.pageEnd}` : `${hit.documentTitle}, p. ${hit.pageStart}`;
|
|
}
|
|
|
|
/**
|
|
* Chooses source material for a generation job and splits it into `groups` batches that
|
|
* each fit `budgetTokens`.
|
|
* - With a focus, the best-matching chunks are used (kept in document order).
|
|
* - Without one, the whole material is used; if it is too long, chunks are sampled evenly
|
|
* so every part of the document is represented.
|
|
*/
|
|
export function selectContext(db: DB, scope: ChunkScope, opts: { focus?: string; budgetTokens: number; groups: number }): ChunkHit[][] {
|
|
const { sql, params } = scopeSql(scope);
|
|
if (!sql) return [];
|
|
const all = db
|
|
.all<ChunkJoin>(
|
|
`SELECT ch.id, ch.document_id, d.title AS doc_title, ch.seq, ch.text, ch.heading, ch.page_start, ch.page_end, ch.tokens
|
|
FROM chunks ch JOIN documents d ON d.id = ch.document_id WHERE 1 = 1 ${sql} ORDER BY d.sort_order, d.created_at, ch.seq`,
|
|
...params,
|
|
)
|
|
.map(mapHit);
|
|
if (all.length === 0) return [];
|
|
const capacity = opts.budgetTokens * opts.groups;
|
|
const total = all.reduce((n, c) => n + c.tokens, 0);
|
|
let chosen = all;
|
|
|
|
if (opts.focus?.trim()) {
|
|
const ranked = searchChunks(db, opts.focus, scope, 400);
|
|
if (ranked.length >= 3) {
|
|
const keep = new Set<number>();
|
|
let used = 0;
|
|
for (const hit of ranked) {
|
|
if (used + hit.tokens > capacity) break;
|
|
keep.add(hit.id);
|
|
used += hit.tokens;
|
|
}
|
|
chosen = all.filter((c) => keep.has(c.id));
|
|
}
|
|
}
|
|
if (chosen === all && total > capacity) {
|
|
// Even sampling: walk the document and keep chunks at a steady stride.
|
|
const stride = total / capacity;
|
|
const picked: ChunkHit[] = [];
|
|
let acc = 0;
|
|
let next = 0;
|
|
for (const c of all) {
|
|
if (acc >= next) {
|
|
picked.push(c);
|
|
next += c.tokens * stride;
|
|
}
|
|
acc += c.tokens;
|
|
}
|
|
chosen = picked;
|
|
}
|
|
|
|
// Contiguous groups of roughly equal size, none above the budget.
|
|
const chosenTotal = chosen.reduce((n, c) => n + c.tokens, 0);
|
|
const groupCount = Math.max(1, Math.min(opts.groups, chosen.length));
|
|
const target = Math.min(opts.budgetTokens, Math.ceil(chosenTotal / groupCount));
|
|
const groups: ChunkHit[][] = [];
|
|
let current: ChunkHit[] = [];
|
|
let size = 0;
|
|
for (const c of chosen) {
|
|
if (current.length > 0 && size + c.tokens > target && groups.length < groupCount - 1) {
|
|
groups.push(current);
|
|
current = [];
|
|
size = 0;
|
|
}
|
|
if (size + c.tokens > opts.budgetTokens && current.length > 0) continue; // last group is full
|
|
current.push(c);
|
|
size += c.tokens;
|
|
}
|
|
if (current.length > 0) groups.push(current);
|
|
return groups;
|
|
}
|
|
|
|
export function formatExcerpts(chunks: ChunkHit[]): string {
|
|
return chunks.map((c) => `[c${c.id}] ${sourceLabel(c)}${c.heading ? ` — ${c.heading}` : ''}\n${c.text}`).join('\n\n---\n\n');
|
|
}
|
|
|
|
export { estimateTokens };
|