import { DatabaseSync } from "node:sqlite"; import { titleCase } from "../../shared/products.js"; // Each source lives in its own table so a refresh can be built alongside the // old copy and swapped in atomically. The file is a cache: it can be deleted // at any time and is rebuilt on the next import. export const sources = ["usda", "off", "api"]; const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT, serving_qty REAL, serving_unit TEXT, nutrients TEXT`; export function openProducts(filename) { const db = new DatabaseSync(filename); db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000; CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT, imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`); for (const source of ["usda", "off"]) db.exec( `CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`, ); // API answers include misses (nutrients NULL) so repeated scans of an // unknown code don't hit Open Food Facts again. db.exec( `CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`, ); const select = Object.fromEntries( sources.map((source) => [ source, db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`), ]), ); const toProduct = (row, source) => row?.nutrients ? { gtin: row.gtin, name: source === "usda" ? titleCase(row.name) : row.name, brand: source === "usda" ? titleCase(row.brand) : row.brand, serving: row.serving, servingQty: row.serving_qty, servingUnit: row.serving_unit, nutrients: JSON.parse(row.nutrients), source, } : null; /** The best local match, or `undefined` when no source has an answer. */ function lookup( gtin, { apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {}, ) { const found = {}; for (const source of sources) found[source] = select[source].get(gtin); const api = found.api; if ( api && Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge) ) found.api = undefined; const usda = toProduct(found.usda, "usda"); const off = toProduct(found.off, "off") || toProduct(found.api, "api"); // USDA comes from manufacturers' own label submissions, so it wins; // community-entered Open Food Facts fills in what it lacks. const base = usda || off; if (base) return { ...base, name: base.name || off?.name, brand: base.brand || off?.brand, }; return found.api ? null : undefined; } const upsertApi = db.prepare( `INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`, ); function remember(gtin, product) { upsertApi.run( gtin, product?.name ?? null, product?.brand ?? null, product?.serving ?? null, product?.servingQty ?? null, product?.servingUnit ?? null, product ? JSON.stringify(product.nutrients) : null, Date.now(), ); } /** Bulk loader used by the importers: writes `_products_next`. */ function beginImport(source) { db.exec(`DROP TABLE IF EXISTS ${source}_products_next; CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`); const insert = db.prepare( `INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`, ); let pending = 0; let rows = 0; db.exec("BEGIN"); return { add(product) { insert.run( product.gtin, product.name || null, product.brand || null, product.serving || null, product.servingQty ?? null, product.servingUnit || null, JSON.stringify(product.nutrients), ); rows++; if (++pending >= 20000) { db.exec("COMMIT; BEGIN"); pending = 0; } }, commit(version) { db.exec(`COMMIT; BEGIN IMMEDIATE; DROP TABLE IF EXISTS ${source}_products; ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`); db.prepare( `INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error) VALUES (?,?,?,?,?,NULL)`, ).run( source, version, new Date().toISOString(), rows, new Date().toISOString(), ); db.exec("COMMIT"); // Old pages are reused by the next import; reclaiming them would // rewrite the whole file. db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); return rows; }, abort() { if (db.isTransaction) db.exec("ROLLBACK"); db.exec(`DROP TABLE IF EXISTS ${source}_products_next`); }, }; } const state = (source) => db.prepare("SELECT * FROM imports WHERE source=?").get(source); function recordCheck(source, error = null) { db.prepare( `INSERT INTO imports(source,checked_at,error) VALUES (?,?,?) ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`, ).run(source, new Date().toISOString(), error); } const status = () => db .prepare("SELECT source, version, imported_at, rows, error FROM imports") .all(); return { db, lookup, remember, beginImport, state, recordCheck, status, close: () => db.close(), }; }