All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 30s
Settings → Data → Import gets "Restore a full backup" in place of the V1 import row. The zip from "Download backup" is uploaded and replaces the database, every setting and the uploaded files on the running server: - the staged database is upgraded to this schema (newer backups are refused), saved keys are re-encrypted for this server's key, interrupted jobs are marked, and the restoring session is carried over; - a pre-restore snapshot is written, running generations stop, and the file is swapped under the open DB handle (DB.replaceWith); uploads and media are copied into place and background work restarts. The /api/import/v1 endpoint stays for the V1 cutover (docs/MIGRATION.md). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
358 lines
15 KiB
TypeScript
358 lines
15 KiB
TypeScript
/**
|
|
* Semantic search over document passages.
|
|
*
|
|
* Each chunk gets an embedding from the configured provider (OpenAI, Gemini, Ollama through the
|
|
* OpenAI-compatible connection, or any
|
|
* OpenAI-compatible server). Vectors are normalised Float32 BLOBs in `chunk_vectors`, keyed by
|
|
* provider + model + size, and searched by brute-force cosine similarity over an in-memory
|
|
* copy: a personal library has thousands of passages, not millions, so no index is needed.
|
|
*
|
|
* Search is hybrid: keyword (FTS5 BM25) and semantic rankings are merged with reciprocal-rank
|
|
* fusion, so exact terms (drug names, lab values) still win while paraphrased questions
|
|
* ("why do patients on furosemide need bananas?") also find the right passage.
|
|
*/
|
|
import { z } from 'zod';
|
|
import { LOCAL_EMBEDDING_LABEL, LOCAL_EMBEDDING_MODEL, LOCAL_PROVIDER_ID, localEmbed, localModelStatus } from '../ai/localEmbed.ts';
|
|
import { getAdapter } from '../ai/providers/index.ts';
|
|
import { providerConfigById } from '../ai/service.ts';
|
|
import { AIError, type ProviderKind } from '../ai/types.ts';
|
|
import type { DB } from '../db.ts';
|
|
import { badRequest, type AppContext } from '../http.ts';
|
|
import { getSetting, setSetting } from '../settings.ts';
|
|
import { now } from '../util.ts';
|
|
import { chunkById, searchChunks, type ChunkHit, type ChunkScope } from './documents.ts';
|
|
|
|
/* ------------------------------ settings ------------------------------ */
|
|
|
|
export interface EmbeddingSettings {
|
|
/** A provider id, `local` for the built-in model, or null for keyword search only. */
|
|
providerId: string | null;
|
|
model: string;
|
|
/** Requested vector size for models that support shortening (OpenAI v3, Gemini). Null = native. */
|
|
dimensions: number | null;
|
|
}
|
|
|
|
const SETTINGS_KEY = 'embeddings';
|
|
const EMPTY: EmbeddingSettings = { providerId: null, model: '', dimensions: null };
|
|
const LOCAL: EmbeddingSettings = { providerId: LOCAL_PROVIDER_ID, model: LOCAL_EMBEDDING_MODEL, dimensions: null };
|
|
|
|
/** Whether an install that never chose an embedding provider uses the built-in model. */
|
|
let localDefault = false;
|
|
export function setLocalEmbeddingDefault(on: boolean): void {
|
|
localDefault = on;
|
|
}
|
|
|
|
/** Suggested embedding model per provider type. Anthropic has no embeddings API. */
|
|
export const DEFAULT_EMBEDDING_MODELS: Record<ProviderKind, string | null> = {
|
|
openai: 'text-embedding-3-small',
|
|
gemini: 'gemini-embedding-001',
|
|
openai_compatible: '',
|
|
openrouter: null,
|
|
anthropic: null,
|
|
deepseek: null,
|
|
};
|
|
|
|
/** USD per million input tokens, for the usage log. */
|
|
const PRICES: Record<string, number> = {
|
|
'text-embedding-3-small': 0.02,
|
|
'text-embedding-3-large': 0.13,
|
|
'text-embedding-ada-002': 0.1,
|
|
'gemini-embedding-001': 0.15,
|
|
};
|
|
|
|
export const embeddingSettingsSchema = z.object({
|
|
providerId: z.string().nullable(),
|
|
model: z.string().trim().max(200).default(''),
|
|
dimensions: z.number().int().min(64).max(4096).nullable().default(null),
|
|
});
|
|
|
|
export function getEmbeddingSettings(db: DB): EmbeddingSettings {
|
|
const stored = getSetting<Partial<EmbeddingSettings> | null>(db, SETTINGS_KEY, null);
|
|
// Nothing chosen yet: the built-in model, so semantic search works without a cloud account.
|
|
if (!stored) return localDefault ? { ...LOCAL } : { ...EMPTY };
|
|
const s = { ...EMPTY, ...stored };
|
|
return s.providerId === LOCAL_PROVIDER_ID ? { ...LOCAL } : s;
|
|
}
|
|
|
|
const modelKey = (s: EmbeddingSettings) => `${s.providerId}:${s.model}:${s.dimensions ?? 'native'}`;
|
|
const isConfigured = (s: EmbeddingSettings) => Boolean(s.providerId && s.model);
|
|
|
|
export function saveEmbeddingSettings(app: AppContext, input: z.input<typeof embeddingSettingsSchema>): EmbeddingSettings {
|
|
const next = embeddingSettingsSchema.parse(input);
|
|
if (next.providerId === LOCAL_PROVIDER_ID) {
|
|
next.model = LOCAL_EMBEDDING_MODEL;
|
|
next.dimensions = null;
|
|
} else if (next.providerId) {
|
|
const { kind } = providerConfigById(app, next.providerId);
|
|
if (!getAdapter(kind).embed) throw badRequest('This provider has no embeddings API. Choose OpenAI, Gemini or an OpenAI-compatible server (Ollama included).');
|
|
if (!next.model) throw badRequest('Enter the embedding model to use.');
|
|
}
|
|
const prev = getEmbeddingSettings(app.db);
|
|
setSetting(app.db, SETTINGS_KEY, next);
|
|
if (modelKey(prev) !== modelKey(next)) {
|
|
// Vectors from different models are not comparable: start over.
|
|
app.db.run('DELETE FROM chunk_vectors WHERE model_key != ?', modelKey(next));
|
|
cache = null;
|
|
state.lastError = '';
|
|
}
|
|
if (isConfigured(next)) scheduleIndexing(app);
|
|
return next;
|
|
}
|
|
|
|
/* ------------------------------ embedding calls ------------------------------ */
|
|
|
|
/** Inputs per request and a cap on characters per input (about 2k tokens). */
|
|
const BATCH = 64;
|
|
const MAX_INPUT_CHARS = 8000;
|
|
|
|
function normalise(v: number[]): Float32Array {
|
|
const out = new Float32Array(v.length);
|
|
let sum = 0;
|
|
for (const x of v) sum += x * x;
|
|
const len = Math.sqrt(sum) || 1;
|
|
for (let i = 0; i < v.length; i++) out[i] = v[i] / len;
|
|
return out;
|
|
}
|
|
|
|
export async function embedTexts(app: AppContext, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]> {
|
|
const s = getEmbeddingSettings(app.db);
|
|
if (!isConfigured(s)) throw new AIError('not_configured', 'Semantic search is not set up. Choose an embedding model in Settings → AI.');
|
|
if (s.providerId === LOCAL_PROVIDER_ID) {
|
|
// Free and on this machine: nothing to log in the usage table.
|
|
const vectors = await localEmbed(app.config.dataDir, texts, purpose, signal);
|
|
return vectors.map(normalise);
|
|
}
|
|
const { cfg, kind, name } = providerConfigById(app, s.providerId!);
|
|
const adapter = getAdapter(kind);
|
|
if (!adapter.embed) throw new AIError('unsupported', `${name} has no embeddings API.`);
|
|
const out: Float32Array[] = [];
|
|
for (let i = 0; i < texts.length; i += BATCH) {
|
|
const inputs = texts.slice(i, i + BATCH).map((t) => t.slice(0, MAX_INPUT_CHARS) || ' ');
|
|
const started = now();
|
|
try {
|
|
const res = await adapter.embed(cfg, { model: s.model, inputs, purpose, dimensions: s.dimensions ?? undefined, signal });
|
|
out.push(...res.vectors.map(normalise));
|
|
logUsage(app.db, s, name, purpose, res.inputTokens, started);
|
|
} catch (err) {
|
|
logUsage(app.db, s, name, purpose, 0, started, err instanceof Error ? err.message : String(err));
|
|
throw err;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function logUsage(db: DB, s: EmbeddingSettings, providerName: string, purpose: string, tokens: number, started: number, error?: string) {
|
|
try {
|
|
const price = PRICES[s.model.replace(/^models\//, '')];
|
|
db.run(
|
|
`INSERT INTO ai_usage (at, provider_id, provider_name, model, task, input_tokens, output_tokens, cache_read_tokens, cost, ok, error, duration_ms)
|
|
VALUES (?, ?, ?, ?, ?, ?, 0, 0, ?, ?, ?, ?)`,
|
|
now(),
|
|
s.providerId,
|
|
providerName,
|
|
s.model,
|
|
purpose === 'query' ? 'semantic search' : 'semantic indexing',
|
|
tokens,
|
|
price !== undefined ? (tokens * price) / 1_000_000 : null,
|
|
error ? 0 : 1,
|
|
error ? error.slice(0, 500) : '',
|
|
now() - started,
|
|
);
|
|
} catch {
|
|
/* usage logging must never break search */
|
|
}
|
|
}
|
|
|
|
/* ------------------------------ background indexing ------------------------------ */
|
|
|
|
const state = { running: false, lastError: '', lastRunAt: 0 };
|
|
|
|
/** Embeds every chunk that has no vector for the current model. Safe to call often. */
|
|
export function scheduleIndexing(app: AppContext): void {
|
|
if (state.running || !isConfigured(getEmbeddingSettings(app.db))) return;
|
|
state.running = true;
|
|
state.lastError = '';
|
|
void (async () => {
|
|
try {
|
|
for (;;) {
|
|
const s = getEmbeddingSettings(app.db);
|
|
if (!isConfigured(s)) break;
|
|
const key = modelKey(s);
|
|
const rows = app.db.all<{ id: number; text: string; heading: string; title: string }>(
|
|
`SELECT ch.id, ch.text, ch.heading, d.title FROM chunks ch JOIN documents d ON d.id = ch.document_id
|
|
LEFT JOIN chunk_vectors v ON v.chunk_id = ch.id AND v.model_key = ?
|
|
WHERE v.chunk_id IS NULL ORDER BY ch.id LIMIT ?`,
|
|
key,
|
|
BATCH,
|
|
);
|
|
if (rows.length === 0) break;
|
|
// The document title and heading give a passage its context ("Digoxin" under "Cardiac drugs").
|
|
const vectors = await embedTexts(app, rows.map((r) => [r.title, r.heading, r.text].filter(Boolean).join('\n')), 'document');
|
|
if (modelKey(getEmbeddingSettings(app.db)) !== key) continue; // settings changed mid-batch
|
|
app.db.tx(() => {
|
|
rows.forEach((r, i) => {
|
|
const v = vectors[i];
|
|
// The chunk may have been deleted while it was being embedded.
|
|
if (app.db.get('SELECT 1 FROM chunks WHERE id = ?', r.id)) {
|
|
app.db.run('INSERT OR REPLACE INTO chunk_vectors (chunk_id, model_key, dims, vector) VALUES (?, ?, ?, ?)', r.id, key, v.length, new Uint8Array(v.buffer, v.byteOffset, v.byteLength));
|
|
}
|
|
});
|
|
});
|
|
}
|
|
} catch (err) {
|
|
state.lastError = err instanceof Error ? err.message : String(err);
|
|
app.log('warn', `Semantic indexing paused: ${state.lastError}`);
|
|
} finally {
|
|
state.running = false;
|
|
state.lastRunAt = now();
|
|
}
|
|
})();
|
|
}
|
|
|
|
/** Waits for the current indexing run. Used by tests. */
|
|
export async function indexingIdle(): Promise<void> {
|
|
while (state.running) await new Promise((r) => setTimeout(r, 10));
|
|
}
|
|
|
|
export function rebuildIndex(app: AppContext): void {
|
|
app.db.run('DELETE FROM chunk_vectors');
|
|
cache = null;
|
|
state.lastError = '';
|
|
scheduleIndexing(app);
|
|
}
|
|
|
|
export function embeddingStatus(app: AppContext) {
|
|
const s = getEmbeddingSettings(app.db);
|
|
const total = app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunks')?.n ?? 0;
|
|
const indexed = isConfigured(s) ? (app.db.get<{ n: number }>('SELECT COUNT(*) AS n FROM chunk_vectors WHERE model_key = ?', modelKey(s))?.n ?? 0) : 0;
|
|
return {
|
|
settings: s,
|
|
configured: isConfigured(s),
|
|
indexed,
|
|
total,
|
|
running: state.running,
|
|
lastError: state.lastError,
|
|
defaults: DEFAULT_EMBEDDING_MODELS,
|
|
local: { ...localModelStatus(app.config.dataDir), label: LOCAL_EMBEDDING_LABEL, enabledByDefault: localDefault },
|
|
};
|
|
}
|
|
|
|
/* ------------------------------ vector search ------------------------------ */
|
|
|
|
interface VectorCache {
|
|
key: string;
|
|
count: number;
|
|
maxId: number;
|
|
dims: number;
|
|
ids: Int32Array;
|
|
docIds: string[];
|
|
classIds: string[];
|
|
matrix: Float32Array;
|
|
}
|
|
let cache: VectorCache | null = null;
|
|
|
|
/** Drops the in-memory vectors (after a restore replaced the database). */
|
|
export function resetVectorCache(): void {
|
|
cache = null;
|
|
queryCache.clear();
|
|
}
|
|
|
|
function loadVectors(db: DB, key: string): VectorCache | null {
|
|
const sig = db.get<{ n: number; m: number | null }>('SELECT COUNT(*) AS n, MAX(chunk_id) AS m FROM chunk_vectors WHERE model_key = ?', key)!;
|
|
if (sig.n === 0) return null;
|
|
if (cache && cache.key === key && cache.count === sig.n && cache.maxId === (sig.m ?? 0)) return cache;
|
|
const rows = db.all<{ chunk_id: number; document_id: string; class_id: string; dims: number; vector: Uint8Array }>(
|
|
`SELECT v.chunk_id, ch.document_id, ch.class_id, v.dims, v.vector FROM chunk_vectors v JOIN chunks ch ON ch.id = v.chunk_id WHERE v.model_key = ?`,
|
|
key,
|
|
);
|
|
const dims = rows[0]?.dims ?? 0;
|
|
const usable = rows.filter((r) => r.dims === dims && r.vector.byteLength === dims * 4);
|
|
const matrix = new Float32Array(usable.length * dims);
|
|
usable.forEach((r, i) => matrix.set(new Float32Array(r.vector.slice().buffer), i * dims));
|
|
cache = {
|
|
key,
|
|
count: sig.n,
|
|
maxId: sig.m ?? 0,
|
|
dims,
|
|
ids: Int32Array.from(usable.map((r) => r.chunk_id)),
|
|
docIds: usable.map((r) => r.document_id),
|
|
classIds: usable.map((r) => r.class_id),
|
|
matrix,
|
|
};
|
|
return cache;
|
|
}
|
|
|
|
const queryCache = new Map<string, Float32Array>();
|
|
|
|
async function queryVector(app: AppContext, key: string, query: string): Promise<Float32Array> {
|
|
const k = `${key}\n${query}`;
|
|
const hit = queryCache.get(k);
|
|
if (hit) return hit;
|
|
const [v] = await embedTexts(app, [query], 'query');
|
|
queryCache.set(k, v);
|
|
if (queryCache.size > 200) queryCache.delete(queryCache.keys().next().value!);
|
|
return v;
|
|
}
|
|
|
|
/** Passages closest in meaning to the query, best first. Empty when semantic search is off. */
|
|
export async function semanticSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8, minScore = 0.2): Promise<(ChunkHit & { similarity: number })[]> {
|
|
const s = getEmbeddingSettings(app.db);
|
|
if (!isConfigured(s) || !query.trim()) return [];
|
|
const key = modelKey(s);
|
|
const vectors = loadVectors(app.db, key);
|
|
if (!vectors) return [];
|
|
const q = await queryVector(app, key, query.trim().slice(0, 2000));
|
|
if (q.length !== vectors.dims) return [];
|
|
const docs = scope.documentIds?.length ? new Set(scope.documentIds) : null;
|
|
const top: { i: number; score: number }[] = [];
|
|
const { dims, matrix } = vectors;
|
|
for (let i = 0; i < vectors.ids.length; i++) {
|
|
if (docs ? !docs.has(vectors.docIds[i]) : scope.classId && vectors.classIds[i] !== scope.classId) continue;
|
|
let dot = 0;
|
|
const off = i * dims;
|
|
for (let d = 0; d < dims; d++) dot += matrix[off + d] * q[d];
|
|
if (dot < minScore) continue;
|
|
if (top.length < limit) {
|
|
top.push({ i, score: dot });
|
|
top.sort((a, b) => b.score - a.score);
|
|
} else if (dot > top[top.length - 1].score) {
|
|
top[top.length - 1] = { i, score: dot };
|
|
top.sort((a, b) => b.score - a.score);
|
|
}
|
|
}
|
|
const out: (ChunkHit & { similarity: number })[] = [];
|
|
for (const t of top) {
|
|
const hit = chunkById(app.db, vectors.ids[t.i]);
|
|
if (hit) out.push({ ...hit, similarity: t.score });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Keyword + semantic search merged by reciprocal-rank fusion. Falls back to keyword search alone
|
|
* when semantic search is off or its provider fails, so search never breaks.
|
|
*/
|
|
export async function hybridSearch(app: AppContext, query: string, scope: ChunkScope, limit = 8): Promise<ChunkHit[]> {
|
|
const pool = Math.max(limit * 3, 30);
|
|
const lexical = searchChunks(app.db, query, scope, pool);
|
|
let semantic: ChunkHit[] = [];
|
|
try {
|
|
semantic = await semanticSearch(app, query, scope, pool);
|
|
} catch (err) {
|
|
app.log('warn', `Semantic search unavailable, using keywords only: ${err instanceof Error ? err.message : String(err)}`);
|
|
}
|
|
if (semantic.length === 0) return lexical.slice(0, limit);
|
|
const K = 60;
|
|
const fused = new Map<number, { hit: ChunkHit; score: number }>();
|
|
const add = (list: ChunkHit[]) =>
|
|
list.forEach((hit, rank) => {
|
|
const prev = fused.get(hit.id);
|
|
fused.set(hit.id, { hit: prev?.hit ?? hit, score: (prev?.score ?? 0) + 1 / (K + rank + 1) });
|
|
});
|
|
add(lexical);
|
|
add(semantic);
|
|
return [...fused.values()]
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, limit)
|
|
.map((x) => ({ ...x.hit, score: x.score }));
|
|
}
|