Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f6981c8a2
commit
80fc48b9f1
39 changed files with 4434 additions and 33 deletions
94
server/products/import.js
Normal file
94
server/products/import.js
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
// Refreshes the local product databases. Runs as a low-priority child process
|
||||
// (see scheduler.js) so parsing several gigabytes never stalls the app.
|
||||
// node server/products/import.js <data dir> [--force]
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { setPriority } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { openProducts } from "./db.js";
|
||||
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
|
||||
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
|
||||
|
||||
const dataDir = resolve(process.argv[2] || "data");
|
||||
const force = process.argv.includes("--force");
|
||||
const OFF_MAX_AGE = 7 * 86400000;
|
||||
const log = (...args) => console.log("[products]", ...args);
|
||||
try {
|
||||
setPriority(19);
|
||||
} catch {
|
||||
// Lowering priority is best effort.
|
||||
}
|
||||
const products = openProducts(join(dataDir, "products.sqlite"));
|
||||
const tmp = join(dataDir, "tmp");
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
|
||||
async function refreshUsda() {
|
||||
const latest = await latestUsdaArchive();
|
||||
if (!force && products.state("usda")?.version === latest.version)
|
||||
return products.recordCheck("usda");
|
||||
const file = join(tmp, "usda-branded.zip");
|
||||
log(`downloading USDA Branded Foods ${latest.version}`);
|
||||
try {
|
||||
await download(latest.url, file);
|
||||
const sink = products.beginImport("usda");
|
||||
try {
|
||||
await importUsdaZip(file, sink, {
|
||||
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
rmSync(file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshOff() {
|
||||
const state = products.state("off");
|
||||
if (
|
||||
!force &&
|
||||
state?.imported_at &&
|
||||
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
|
||||
)
|
||||
return;
|
||||
const response = await fetch(OFF_CSV, {
|
||||
headers: { "User-Agent": userAgent },
|
||||
});
|
||||
if (!response.ok)
|
||||
throw new Error(`Open Food Facts answered ${response.status}`);
|
||||
const version =
|
||||
response.headers.get("last-modified") || new Date().toUTCString();
|
||||
if (!force && state?.version === version) {
|
||||
await response.body?.cancel();
|
||||
return products.recordCheck("off");
|
||||
}
|
||||
log("streaming the Open Food Facts export");
|
||||
const sink = products.beginImport("off");
|
||||
try {
|
||||
await importOffCsv(response, sink, {
|
||||
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [source, refresh] of [
|
||||
["usda", refreshUsda],
|
||||
["off", refreshOff],
|
||||
]) {
|
||||
try {
|
||||
await refresh();
|
||||
} catch (error) {
|
||||
console.error(`[products] ${source} refresh failed:`, error.message);
|
||||
products.recordCheck(source, String(error.message).slice(0, 300));
|
||||
}
|
||||
}
|
||||
products.close();
|
||||
Loading…
Add table
Add a link
Reference in a new issue