/** * Built-in embedding model, used for semantic search when no cloud embedding provider is * configured. Runs on the server's CPU through ONNX Runtime (transformers.js), so it needs no * API key and costs nothing. * * Model: BAAI bge-small-en-v1.5 (ONNX, 8-bit quantised, about 34 MB, 384 dimensions). It is a * strong small retrieval model; queries get BGE's recommended instruction prefix. The files * are downloaded from Hugging Face on first use into DATA_DIR/models and reused afterwards. * * The library is imported lazily: the app starts (and tests run) without loading ONNX Runtime * until something actually needs an embedding. * * The Docker image ships the model in /app/models (LOCAL_MODELS_DIR), so nothing is downloaded * at runtime there. Elsewhere it is downloaded once into DATA_DIR/models. */ import fs from 'node:fs'; import path from 'node:path'; import { AIError } from './types.ts'; export const LOCAL_PROVIDER_ID = 'local'; export const LOCAL_EMBEDDING_MODEL = 'Xenova/bge-small-en-v1.5'; export const LOCAL_EMBEDDING_LABEL = 'bge-small-en-v1.5 (built in)'; export const LOCAL_EMBEDDING_DIMS = 384; const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: '; /** Inputs per forward pass: small enough to keep memory flat on a home server. */ const BATCH = 16; /** About 512 tokens, the model's window. Longer input is truncated by the tokenizer anyway. */ const MAX_CHARS = 2000; type Extractor = (texts: string[], opts: { pooling: 'cls' | 'mean'; normalize: boolean }) => Promise<{ tolist(): number[][] }>; const state: { extractor: Promise | null; cacheDir: string; status: 'idle' | 'loading' | 'ready' | 'error'; error: string } = { extractor: null, cacheDir: '', status: 'idle', error: '', }; export function localModelDir(dataDir: string): string { return path.join(dataDir, 'models'); } function hasModel(root: string | undefined): boolean { if (!root) return false; const dir = path.join(root, ...LOCAL_EMBEDDING_MODEL.split('/')); try { return fs.existsSync(path.join(dir, 'onnx', 'model_quantized.onnx')) && fs.existsSync(path.join(dir, 'tokenizer.json')); } catch { return false; } } /** A copy of the model shipped with the installation (the Docker image), if any. */ function bundledModelDir(): string | undefined { const dir = process.env.LOCAL_MODELS_DIR?.trim(); return dir && hasModel(dir) ? dir : undefined; } /** True when the model files are already on disk (no download needed). */ export function localModelDownloaded(dataDir: string): boolean { return Boolean(bundledModelDir()) || hasModel(localModelDir(dataDir)); } /** * transformers.js imports `sharp` (image processing) when it loads, although text embeddings * never use it. sharp's prebuilt binary needs a fairly recent x86-64 CPU; on hosts where it * cannot load, the whole library failed. Route `sharp` to an empty stand-in in that case. */ async function stubSharpIfBroken(): Promise { try { await import('sharp'); return; } catch { /* fall through */ } const { registerHooks } = (await import('node:module')) as unknown as { registerHooks?: (hooks: unknown) => void }; const stub = 'data:text/javascript,' + encodeURIComponent('export default function sharp(){throw new Error("Image processing is not available on this server.")}'); registerHooks?.({ resolve(specifier: string, context: unknown, next: (s: string, c: unknown) => unknown) { if (specifier === 'sharp') return { url: stub, shortCircuit: true, format: 'module' }; return next(specifier, context); }, }); } export function localModelStatus(dataDir: string) { return { model: LOCAL_EMBEDDING_MODEL, label: LOCAL_EMBEDDING_LABEL, dimensions: LOCAL_EMBEDDING_DIMS, downloaded: localModelDownloaded(dataDir), status: state.status, error: state.error }; } async function load(dataDir: string): Promise { const cacheDir = localModelDir(dataDir); if (state.extractor && state.cacheDir === cacheDir) return state.extractor; state.cacheDir = cacheDir; state.status = 'loading'; state.error = ''; state.extractor = (async () => { const bundled = bundledModelDir(); if (!bundled) fs.mkdirSync(cacheDir, { recursive: true }); let lib: typeof import('@huggingface/transformers'); try { await stubSharpIfBroken(); lib = await import('@huggingface/transformers'); } catch (err) { throw new AIError('unsupported', `The built-in embedding model could not start on this server: ${err instanceof Error ? err.message : String(err)}`, { cause: err }); } lib.env.cacheDir = cacheDir; lib.env.allowLocalModels = true; lib.env.localModelPath = bundled ?? cacheDir; // With a bundled copy there is nothing to download. lib.env.allowRemoteModels = !bundled; const extractor = await lib.pipeline('feature-extraction', LOCAL_EMBEDDING_MODEL, { dtype: 'q8' }); return extractor as unknown as Extractor; })(); try { const ready = await state.extractor; state.status = 'ready'; return ready; } catch (err) { state.extractor = null; state.status = 'error'; const message = err instanceof AIError ? err.message : `The built-in embedding model could not be loaded: ${err instanceof Error ? err.message : String(err)}`; state.error = message; throw err instanceof AIError ? err : new AIError('network', `${message}. It is downloaded once (about 35 MB) from huggingface.co; check that the server can reach the internet.`, { cause: err, retryable: true }); } } /** Embeds texts with the built-in model. Vectors come back L2-normalised. */ export async function localEmbed(dataDir: string, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise { const extractor = await load(dataDir); const out: number[][] = []; for (let i = 0; i < texts.length; i += BATCH) { if (signal?.aborted) throw new AIError('aborted', 'The request was cancelled.'); const batch = texts.slice(i, i + BATCH).map((t) => (purpose === 'query' ? QUERY_PREFIX : '') + (t.slice(0, MAX_CHARS) || ' ')); const tensor = await extractor(batch, { pooling: 'cls', normalize: true }); out.push(...tensor.tolist()); // Let the event loop serve requests between batches; indexing runs in the background. await new Promise((resolve) => setImmediate(resolve)); } return out; }