/** * Built-in embedding model, used for semantic search when no cloud embedding provider is * configured. Runs on the server's CPU through ONNX Runtime (transformers.js), so it needs no * API key and costs nothing. * * Model: BAAI bge-small-en-v1.5 (ONNX, 8-bit quantised, about 34 MB, 384 dimensions). It is a * strong small retrieval model; queries get BGE's recommended instruction prefix. The files * are downloaded from Hugging Face on first use into DATA_DIR/models and reused afterwards. * * The library is imported lazily: the app starts (and tests run) without loading ONNX Runtime * until something actually needs an embedding. */ import fs from 'node:fs'; import path from 'node:path'; import { AIError } from './types.ts'; export const LOCAL_PROVIDER_ID = 'local'; export const LOCAL_EMBEDDING_MODEL = 'Xenova/bge-small-en-v1.5'; export const LOCAL_EMBEDDING_LABEL = 'bge-small-en-v1.5 (built in)'; export const LOCAL_EMBEDDING_DIMS = 384; const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: '; /** Inputs per forward pass: small enough to keep memory flat on a home server. */ const BATCH = 16; /** About 512 tokens, the model's window. Longer input is truncated by the tokenizer anyway. */ const MAX_CHARS = 2000; type Extractor = (texts: string[], opts: { pooling: 'cls' | 'mean'; normalize: boolean }) => Promise<{ tolist(): number[][] }>; const state: { extractor: Promise | null; cacheDir: string; status: 'idle' | 'loading' | 'ready' | 'error'; error: string } = { extractor: null, cacheDir: '', status: 'idle', error: '', }; export function localModelDir(dataDir: string): string { return path.join(dataDir, 'models'); } /** True when the model files are already on disk (no download needed). */ export function localModelDownloaded(dataDir: string): boolean { const dir = path.join(localModelDir(dataDir), ...LOCAL_EMBEDDING_MODEL.split('/')); try { return fs.existsSync(path.join(dir, 'onnx', 'model_quantized.onnx')) && fs.existsSync(path.join(dir, 'tokenizer.json')); } catch { return false; } } export function localModelStatus(dataDir: string) { return { model: LOCAL_EMBEDDING_MODEL, label: LOCAL_EMBEDDING_LABEL, dimensions: LOCAL_EMBEDDING_DIMS, downloaded: localModelDownloaded(dataDir), status: state.status, error: state.error }; } async function load(dataDir: string): Promise { const cacheDir = localModelDir(dataDir); if (state.extractor && state.cacheDir === cacheDir) return state.extractor; state.cacheDir = cacheDir; state.status = 'loading'; state.error = ''; state.extractor = (async () => { fs.mkdirSync(cacheDir, { recursive: true }); let lib: typeof import('@huggingface/transformers'); try { lib = await import('@huggingface/transformers'); } catch (err) { throw new AIError('unsupported', 'The built-in embedding model is not available in this installation.', { cause: err }); } lib.env.cacheDir = cacheDir; lib.env.allowRemoteModels = true; lib.env.allowLocalModels = true; lib.env.localModelPath = cacheDir; const extractor = await lib.pipeline('feature-extraction', LOCAL_EMBEDDING_MODEL, { dtype: 'q8' }); return extractor as unknown as Extractor; })(); try { const ready = await state.extractor; state.status = 'ready'; return ready; } catch (err) { state.extractor = null; state.status = 'error'; const message = err instanceof AIError ? err.message : `The built-in embedding model could not be loaded: ${err instanceof Error ? err.message : String(err)}`; state.error = message; throw err instanceof AIError ? err : new AIError('network', `${message}. It is downloaded once (about 35 MB) from huggingface.co; check that the server can reach the internet.`, { cause: err, retryable: true }); } } /** Embeds texts with the built-in model. Vectors come back L2-normalised. */ export async function localEmbed(dataDir: string, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise { const extractor = await load(dataDir); const out: number[][] = []; for (let i = 0; i < texts.length; i += BATCH) { if (signal?.aborted) throw new AIError('aborted', 'The request was cancelled.'); const batch = texts.slice(i, i + BATCH).map((t) => (purpose === 'query' ? QUERY_PREFIX : '') + (t.slice(0, MAX_CHARS) || ' ')); const tensor = await extractor(batch, { pooling: 'cls', normalize: true }); out.push(...tensor.tolist()); // Let the event loop serve requests between batches; indexing runs in the background. await new Promise((resolve) => setImmediate(resolve)); } return out; }