// Refreshes the local product databases. Runs as a low-priority child process // (see scheduler.js) so parsing several gigabytes never stalls the app. // node server/products/import.js [--force] import { mkdirSync, rmSync } from "node:fs"; import { setPriority } from "node:os"; import { join, resolve } from "node:path"; import { openProducts } from "./db.js"; import { OFF_CSV, importOffCsv, userAgent } from "./off.js"; import { download, importUsdaZip, latestUsdaArchive } from "./usda.js"; const dataDir = resolve(process.argv[2] || "data"); const force = process.argv.includes("--force"); const OFF_MAX_AGE = 7 * 86400000; const log = (...args) => console.log("[products]", ...args); try { setPriority(19); } catch { // Lowering priority is best effort. } const products = openProducts(join(dataDir, "products.sqlite")); const tmp = join(dataDir, "tmp"); mkdirSync(tmp, { recursive: true }); async function refreshUsda() { const latest = await latestUsdaArchive(); if (!force && products.state("usda")?.version === latest.version) return products.recordCheck("usda"); const file = join(tmp, "usda-branded.zip"); log(`downloading USDA Branded Foods ${latest.version}`); try { await download(latest.url, file); const sink = products.beginImport("usda"); try { await importUsdaZip(file, sink, { onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`), }); log( `USDA: imported ${sink.commit(latest.version).toLocaleString()} products`, ); } catch (error) { sink.abort(); throw error; } } finally { rmSync(file, { force: true }); } } async function refreshOff() { const state = products.state("off"); if ( !force && state?.imported_at && Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE ) return; const response = await fetch(OFF_CSV, { headers: { "User-Agent": userAgent }, }); if (!response.ok) throw new Error(`Open Food Facts answered ${response.status}`); const version = response.headers.get("last-modified") || new Date().toUTCString(); if (!force && state?.version === version) { await response.body?.cancel(); return products.recordCheck("off"); } log("streaming the Open Food Facts export"); const sink = products.beginImport("off"); try { await importOffCsv(response, sink, { onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`), }); log( `Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`, ); } catch (error) { sink.abort(); throw error; } } for (const [source, refresh] of [ ["usda", refreshUsda], ["off", refreshOff], ]) { try { await refresh(); } catch (error) { console.error(`[products] ${source} refresh failed:`, error.message); products.recordCheck(source, String(error.message).slice(0, 300)); } } products.close();