Calio/server/products/import.js
Elijah 80fc48b9f1
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Add barcode and nutrition label scanning
Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 13:41:44 -07:00

94 lines
2.9 KiB
JavaScript

// Refreshes the local product databases. Runs as a low-priority child process
// (see scheduler.js) so parsing several gigabytes never stalls the app.
// node server/products/import.js <data dir> [--force]
import { mkdirSync, rmSync } from "node:fs";
import { setPriority } from "node:os";
import { join, resolve } from "node:path";
import { openProducts } from "./db.js";
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
const dataDir = resolve(process.argv[2] || "data");
const force = process.argv.includes("--force");
const OFF_MAX_AGE = 7 * 86400000;
const log = (...args) => console.log("[products]", ...args);
try {
setPriority(19);
} catch {
// Lowering priority is best effort.
}
const products = openProducts(join(dataDir, "products.sqlite"));
const tmp = join(dataDir, "tmp");
mkdirSync(tmp, { recursive: true });
async function refreshUsda() {
const latest = await latestUsdaArchive();
if (!force && products.state("usda")?.version === latest.version)
return products.recordCheck("usda");
const file = join(tmp, "usda-branded.zip");
log(`downloading USDA Branded Foods ${latest.version}`);
try {
await download(latest.url, file);
const sink = products.beginImport("usda");
try {
await importUsdaZip(file, sink, {
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
});
log(
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
} finally {
rmSync(file, { force: true });
}
}
async function refreshOff() {
const state = products.state("off");
if (
!force &&
state?.imported_at &&
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
)
return;
const response = await fetch(OFF_CSV, {
headers: { "User-Agent": userAgent },
});
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const version =
response.headers.get("last-modified") || new Date().toUTCString();
if (!force && state?.version === version) {
await response.body?.cancel();
return products.recordCheck("off");
}
log("streaming the Open Food Facts export");
const sink = products.beginImport("off");
try {
await importOffCsv(response, sink, {
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
});
log(
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
}
for (const [source, refresh] of [
["usda", refreshUsda],
["off", refreshOff],
]) {
try {
await refresh();
} catch (error) {
console.error(`[products] ${source} refresh failed:`, error.message);
products.recordCheck(source, String(error.message).slice(0, 300));
}
}
products.close();