// Downloads the built-in embedding model into a folder at image build time, so the server // never has to reach huggingface.co at runtime. Usage: node scripts/fetch-embedding-model.mjs import fs from 'node:fs'; import path from 'node:path'; const dir = path.resolve(process.argv[2] ?? 'models'); const MODEL = 'Xenova/bge-small-en-v1.5'; fs.mkdirSync(dir, { recursive: true }); const { env, pipeline } = await import('@huggingface/transformers'); env.cacheDir = dir; env.localModelPath = dir; env.allowRemoteModels = true; const extractor = await pipeline('feature-extraction', MODEL, { dtype: 'q8' }); const [v] = (await extractor(['Furosemide is a loop diuretic.'], { pooling: 'cls', normalize: true })).tolist(); const files = fs.readdirSync(path.join(dir, ...MODEL.split('/')), { recursive: true }); console.log(`Embedding model ready in ${dir}: ${v.length} dimensions, files: ${files.join(', ')}`);