Study/server/ai/localEmbed.ts
Elijah f0aabe4d5d
All checks were successful
Verify and publish V2 container / build-and-push (push) Successful in 1m2s
First revision: DeepSeek, numeric questions, background assistant, UI fixes
Providers and pricing
- Add a DeepSeek provider: thinking mode with low/high/max effort, reasoning_content
  replayed between tool steps, cache-hit usage, /models discovery and catalog prices.
- Fix "does not support tool calling": a rejected forced tool_choice (DeepSeek thinking
  mode) now asks for the call in the prompt; only a genuine no-tools 400 is "unsupported".
- Seven reasoning-effort levels (Off to Max) mapped per adapter.
- Cached-input price per model, cache-write and DeepSeek off-peak aware cost estimates.
- Built-in bge-small embedding model (transformers.js) as the semantic search default.

Quizzes
- Numeric (type-the-answer) questions: units ignored, optional ± tolerance; editor,
  runner, importer, templates and generation prompts.
- Quiz start screen instead of showing answers; navigator ring no longer clipped and
  key legend removed; clock only for timed exams; 7-point letter grades.

Assistant and generation
- Assistant turns run on the server independently of the browser, with re-attach and
  an explicit stop; drafts tray shows generations in progress.
- Per-chat model switcher, @-tagging of class materials, full-width assistant page and
  a smaller assistant button.
- Editable generation instructions and built-in presets, separate from the chatbot
  import instructions.

Other
- Dialog footer padding, keyboard hints removed, copy buttons work over plain HTTP,
  class and folder shares include documents.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 12:48:17 -07:00

101 lines
4.6 KiB
TypeScript

/**
* Built-in embedding model, used for semantic search when no cloud embedding provider is
* configured. Runs on the server's CPU through ONNX Runtime (transformers.js), so it needs no
* API key and costs nothing.
*
* Model: BAAI bge-small-en-v1.5 (ONNX, 8-bit quantised, about 34 MB, 384 dimensions). It is a
* strong small retrieval model; queries get BGE's recommended instruction prefix. The files
* are downloaded from Hugging Face on first use into DATA_DIR/models and reused afterwards.
*
* The library is imported lazily: the app starts (and tests run) without loading ONNX Runtime
* until something actually needs an embedding.
*/
import fs from 'node:fs';
import path from 'node:path';
import { AIError } from './types.ts';
export const LOCAL_PROVIDER_ID = 'local';
export const LOCAL_EMBEDDING_MODEL = 'Xenova/bge-small-en-v1.5';
export const LOCAL_EMBEDDING_LABEL = 'bge-small-en-v1.5 (built in)';
export const LOCAL_EMBEDDING_DIMS = 384;
const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: ';
/** Inputs per forward pass: small enough to keep memory flat on a home server. */
const BATCH = 16;
/** About 512 tokens, the model's window. Longer input is truncated by the tokenizer anyway. */
const MAX_CHARS = 2000;
type Extractor = (texts: string[], opts: { pooling: 'cls' | 'mean'; normalize: boolean }) => Promise<{ tolist(): number[][] }>;
const state: { extractor: Promise<Extractor> | null; cacheDir: string; status: 'idle' | 'loading' | 'ready' | 'error'; error: string } = {
extractor: null,
cacheDir: '',
status: 'idle',
error: '',
};
export function localModelDir(dataDir: string): string {
return path.join(dataDir, 'models');
}
/** True when the model files are already on disk (no download needed). */
export function localModelDownloaded(dataDir: string): boolean {
const dir = path.join(localModelDir(dataDir), ...LOCAL_EMBEDDING_MODEL.split('/'));
try {
return fs.existsSync(path.join(dir, 'onnx', 'model_quantized.onnx')) && fs.existsSync(path.join(dir, 'tokenizer.json'));
} catch {
return false;
}
}
export function localModelStatus(dataDir: string) {
return { model: LOCAL_EMBEDDING_MODEL, label: LOCAL_EMBEDDING_LABEL, dimensions: LOCAL_EMBEDDING_DIMS, downloaded: localModelDownloaded(dataDir), status: state.status, error: state.error };
}
async function load(dataDir: string): Promise<Extractor> {
const cacheDir = localModelDir(dataDir);
if (state.extractor && state.cacheDir === cacheDir) return state.extractor;
state.cacheDir = cacheDir;
state.status = 'loading';
state.error = '';
state.extractor = (async () => {
fs.mkdirSync(cacheDir, { recursive: true });
let lib: typeof import('@huggingface/transformers');
try {
lib = await import('@huggingface/transformers');
} catch (err) {
throw new AIError('unsupported', 'The built-in embedding model is not available in this installation.', { cause: err });
}
lib.env.cacheDir = cacheDir;
lib.env.allowRemoteModels = true;
lib.env.allowLocalModels = true;
lib.env.localModelPath = cacheDir;
const extractor = await lib.pipeline('feature-extraction', LOCAL_EMBEDDING_MODEL, { dtype: 'q8' });
return extractor as unknown as Extractor;
})();
try {
const ready = await state.extractor;
state.status = 'ready';
return ready;
} catch (err) {
state.extractor = null;
state.status = 'error';
const message = err instanceof AIError ? err.message : `The built-in embedding model could not be loaded: ${err instanceof Error ? err.message : String(err)}`;
state.error = message;
throw err instanceof AIError ? err : new AIError('network', `${message}. It is downloaded once (about 35 MB) from huggingface.co; check that the server can reach the internet.`, { cause: err, retryable: true });
}
}
/** Embeds texts with the built-in model. Vectors come back L2-normalised. */
export async function localEmbed(dataDir: string, texts: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<number[][]> {
const extractor = await load(dataDir);
const out: number[][] = [];
for (let i = 0; i < texts.length; i += BATCH) {
if (signal?.aborted) throw new AIError('aborted', 'The request was cancelled.');
const batch = texts.slice(i, i + BATCH).map((t) => (purpose === 'query' ? QUERY_PREFIX : '') + (t.slice(0, MAX_CHARS) || ' '));
const tensor = await extractor(batch, { pooling: 'cls', normalize: true });
out.push(...tensor.tolist());
// Let the event loop serve requests between batches; indexing runs in the background.
await new Promise((resolve) => setImmediate(resolve));
}
return out;
}