All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
2.9 KiB
JavaScript
94 lines
2.9 KiB
JavaScript
// Refreshes the local product databases. Runs as a low-priority child process
|
|
// (see scheduler.js) so parsing several gigabytes never stalls the app.
|
|
// node server/products/import.js <data dir> [--force]
|
|
import { mkdirSync, rmSync } from "node:fs";
|
|
import { setPriority } from "node:os";
|
|
import { join, resolve } from "node:path";
|
|
import { openProducts } from "./db.js";
|
|
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
|
|
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
|
|
|
|
const dataDir = resolve(process.argv[2] || "data");
|
|
const force = process.argv.includes("--force");
|
|
const OFF_MAX_AGE = 7 * 86400000;
|
|
const log = (...args) => console.log("[products]", ...args);
|
|
try {
|
|
setPriority(19);
|
|
} catch {
|
|
// Lowering priority is best effort.
|
|
}
|
|
const products = openProducts(join(dataDir, "products.sqlite"));
|
|
const tmp = join(dataDir, "tmp");
|
|
mkdirSync(tmp, { recursive: true });
|
|
|
|
async function refreshUsda() {
|
|
const latest = await latestUsdaArchive();
|
|
if (!force && products.state("usda")?.version === latest.version)
|
|
return products.recordCheck("usda");
|
|
const file = join(tmp, "usda-branded.zip");
|
|
log(`downloading USDA Branded Foods ${latest.version}`);
|
|
try {
|
|
await download(latest.url, file);
|
|
const sink = products.beginImport("usda");
|
|
try {
|
|
await importUsdaZip(file, sink, {
|
|
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
|
|
});
|
|
log(
|
|
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
|
|
);
|
|
} catch (error) {
|
|
sink.abort();
|
|
throw error;
|
|
}
|
|
} finally {
|
|
rmSync(file, { force: true });
|
|
}
|
|
}
|
|
|
|
async function refreshOff() {
|
|
const state = products.state("off");
|
|
if (
|
|
!force &&
|
|
state?.imported_at &&
|
|
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
|
|
)
|
|
return;
|
|
const response = await fetch(OFF_CSV, {
|
|
headers: { "User-Agent": userAgent },
|
|
});
|
|
if (!response.ok)
|
|
throw new Error(`Open Food Facts answered ${response.status}`);
|
|
const version =
|
|
response.headers.get("last-modified") || new Date().toUTCString();
|
|
if (!force && state?.version === version) {
|
|
await response.body?.cancel();
|
|
return products.recordCheck("off");
|
|
}
|
|
log("streaming the Open Food Facts export");
|
|
const sink = products.beginImport("off");
|
|
try {
|
|
await importOffCsv(response, sink, {
|
|
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
|
|
});
|
|
log(
|
|
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
|
|
);
|
|
} catch (error) {
|
|
sink.abort();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
for (const [source, refresh] of [
|
|
["usda", refreshUsda],
|
|
["off", refreshOff],
|
|
]) {
|
|
try {
|
|
await refresh();
|
|
} catch (error) {
|
|
console.error(`[products] ${source} refresh failed:`, error.message);
|
|
products.recordCheck(source, String(error.message).slice(0, 300));
|
|
}
|
|
}
|
|
products.close();
|