All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
165 lines
5.3 KiB
JavaScript
165 lines
5.3 KiB
JavaScript
import { DatabaseSync } from "node:sqlite";
|
|
import { titleCase } from "../../shared/products.js";
|
|
|
|
// Each source lives in its own table so a refresh can be built alongside the
|
|
// old copy and swapped in atomically. The file is a cache: it can be deleted
|
|
// at any time and is rebuilt on the next import.
|
|
export const sources = ["usda", "off", "api"];
|
|
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
|
|
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
|
|
|
|
export function openProducts(filename) {
|
|
const db = new DatabaseSync(filename);
|
|
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
|
|
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
|
|
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
|
|
for (const source of ["usda", "off"])
|
|
db.exec(
|
|
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
|
|
);
|
|
// API answers include misses (nutrients NULL) so repeated scans of an
|
|
// unknown code don't hit Open Food Facts again.
|
|
db.exec(
|
|
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
|
|
);
|
|
const select = Object.fromEntries(
|
|
sources.map((source) => [
|
|
source,
|
|
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
|
|
]),
|
|
);
|
|
const toProduct = (row, source) =>
|
|
row?.nutrients
|
|
? {
|
|
gtin: row.gtin,
|
|
name: source === "usda" ? titleCase(row.name) : row.name,
|
|
brand: source === "usda" ? titleCase(row.brand) : row.brand,
|
|
serving: row.serving,
|
|
servingQty: row.serving_qty,
|
|
servingUnit: row.serving_unit,
|
|
nutrients: JSON.parse(row.nutrients),
|
|
source,
|
|
}
|
|
: null;
|
|
|
|
/** The best local match, or `undefined` when no source has an answer. */
|
|
function lookup(
|
|
gtin,
|
|
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
|
|
) {
|
|
const found = {};
|
|
for (const source of sources) found[source] = select[source].get(gtin);
|
|
const api = found.api;
|
|
if (
|
|
api &&
|
|
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
|
|
)
|
|
found.api = undefined;
|
|
const usda = toProduct(found.usda, "usda");
|
|
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
|
|
// USDA comes from manufacturers' own label submissions, so it wins;
|
|
// community-entered Open Food Facts fills in what it lacks.
|
|
const base = usda || off;
|
|
if (base)
|
|
return {
|
|
...base,
|
|
name: base.name || off?.name,
|
|
brand: base.brand || off?.brand,
|
|
};
|
|
return found.api ? null : undefined;
|
|
}
|
|
|
|
const upsertApi = db.prepare(
|
|
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
|
|
);
|
|
function remember(gtin, product) {
|
|
upsertApi.run(
|
|
gtin,
|
|
product?.name ?? null,
|
|
product?.brand ?? null,
|
|
product?.serving ?? null,
|
|
product?.servingQty ?? null,
|
|
product?.servingUnit ?? null,
|
|
product ? JSON.stringify(product.nutrients) : null,
|
|
Date.now(),
|
|
);
|
|
}
|
|
|
|
/** Bulk loader used by the importers: writes `<source>_products_next`. */
|
|
function beginImport(source) {
|
|
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
|
|
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
|
|
const insert = db.prepare(
|
|
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
|
|
);
|
|
let pending = 0;
|
|
let rows = 0;
|
|
db.exec("BEGIN");
|
|
return {
|
|
add(product) {
|
|
insert.run(
|
|
product.gtin,
|
|
product.name || null,
|
|
product.brand || null,
|
|
product.serving || null,
|
|
product.servingQty ?? null,
|
|
product.servingUnit || null,
|
|
JSON.stringify(product.nutrients),
|
|
);
|
|
rows++;
|
|
if (++pending >= 20000) {
|
|
db.exec("COMMIT; BEGIN");
|
|
pending = 0;
|
|
}
|
|
},
|
|
commit(version) {
|
|
db.exec(`COMMIT; BEGIN IMMEDIATE;
|
|
DROP TABLE IF EXISTS ${source}_products;
|
|
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
|
|
db.prepare(
|
|
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
|
|
VALUES (?,?,?,?,?,NULL)`,
|
|
).run(
|
|
source,
|
|
version,
|
|
new Date().toISOString(),
|
|
rows,
|
|
new Date().toISOString(),
|
|
);
|
|
db.exec("COMMIT");
|
|
// Old pages are reused by the next import; reclaiming them would
|
|
// rewrite the whole file.
|
|
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
return rows;
|
|
},
|
|
abort() {
|
|
if (db.isTransaction) db.exec("ROLLBACK");
|
|
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
|
|
},
|
|
};
|
|
}
|
|
|
|
const state = (source) =>
|
|
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
|
|
function recordCheck(source, error = null) {
|
|
db.prepare(
|
|
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
|
|
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
|
|
).run(source, new Date().toISOString(), error);
|
|
}
|
|
const status = () =>
|
|
db
|
|
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
|
|
.all();
|
|
|
|
return {
|
|
db,
|
|
lookup,
|
|
remember,
|
|
beginImport,
|
|
state,
|
|
recordCheck,
|
|
status,
|
|
close: () => db.close(),
|
|
};
|
|
}
|