Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f6981c8a2
commit
80fc48b9f1
39 changed files with 4434 additions and 33 deletions
165
server/products/db.js
Normal file
165
server/products/db.js
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { titleCase } from "../../shared/products.js";
|
||||
|
||||
// Each source lives in its own table so a refresh can be built alongside the
|
||||
// old copy and swapped in atomically. The file is a cache: it can be deleted
|
||||
// at any time and is rebuilt on the next import.
|
||||
export const sources = ["usda", "off", "api"];
|
||||
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
|
||||
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
|
||||
|
||||
export function openProducts(filename) {
|
||||
const db = new DatabaseSync(filename);
|
||||
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
|
||||
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
|
||||
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
|
||||
for (const source of ["usda", "off"])
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
|
||||
);
|
||||
// API answers include misses (nutrients NULL) so repeated scans of an
|
||||
// unknown code don't hit Open Food Facts again.
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
|
||||
);
|
||||
const select = Object.fromEntries(
|
||||
sources.map((source) => [
|
||||
source,
|
||||
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
|
||||
]),
|
||||
);
|
||||
const toProduct = (row, source) =>
|
||||
row?.nutrients
|
||||
? {
|
||||
gtin: row.gtin,
|
||||
name: source === "usda" ? titleCase(row.name) : row.name,
|
||||
brand: source === "usda" ? titleCase(row.brand) : row.brand,
|
||||
serving: row.serving,
|
||||
servingQty: row.serving_qty,
|
||||
servingUnit: row.serving_unit,
|
||||
nutrients: JSON.parse(row.nutrients),
|
||||
source,
|
||||
}
|
||||
: null;
|
||||
|
||||
/** The best local match, or `undefined` when no source has an answer. */
|
||||
function lookup(
|
||||
gtin,
|
||||
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
|
||||
) {
|
||||
const found = {};
|
||||
for (const source of sources) found[source] = select[source].get(gtin);
|
||||
const api = found.api;
|
||||
if (
|
||||
api &&
|
||||
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
|
||||
)
|
||||
found.api = undefined;
|
||||
const usda = toProduct(found.usda, "usda");
|
||||
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
|
||||
// USDA comes from manufacturers' own label submissions, so it wins;
|
||||
// community-entered Open Food Facts fills in what it lacks.
|
||||
const base = usda || off;
|
||||
if (base)
|
||||
return {
|
||||
...base,
|
||||
name: base.name || off?.name,
|
||||
brand: base.brand || off?.brand,
|
||||
};
|
||||
return found.api ? null : undefined;
|
||||
}
|
||||
|
||||
const upsertApi = db.prepare(
|
||||
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
|
||||
);
|
||||
function remember(gtin, product) {
|
||||
upsertApi.run(
|
||||
gtin,
|
||||
product?.name ?? null,
|
||||
product?.brand ?? null,
|
||||
product?.serving ?? null,
|
||||
product?.servingQty ?? null,
|
||||
product?.servingUnit ?? null,
|
||||
product ? JSON.stringify(product.nutrients) : null,
|
||||
Date.now(),
|
||||
);
|
||||
}
|
||||
|
||||
/** Bulk loader used by the importers: writes `<source>_products_next`. */
|
||||
function beginImport(source) {
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
|
||||
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
|
||||
const insert = db.prepare(
|
||||
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
|
||||
);
|
||||
let pending = 0;
|
||||
let rows = 0;
|
||||
db.exec("BEGIN");
|
||||
return {
|
||||
add(product) {
|
||||
insert.run(
|
||||
product.gtin,
|
||||
product.name || null,
|
||||
product.brand || null,
|
||||
product.serving || null,
|
||||
product.servingQty ?? null,
|
||||
product.servingUnit || null,
|
||||
JSON.stringify(product.nutrients),
|
||||
);
|
||||
rows++;
|
||||
if (++pending >= 20000) {
|
||||
db.exec("COMMIT; BEGIN");
|
||||
pending = 0;
|
||||
}
|
||||
},
|
||||
commit(version) {
|
||||
db.exec(`COMMIT; BEGIN IMMEDIATE;
|
||||
DROP TABLE IF EXISTS ${source}_products;
|
||||
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
|
||||
db.prepare(
|
||||
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
|
||||
VALUES (?,?,?,?,?,NULL)`,
|
||||
).run(
|
||||
source,
|
||||
version,
|
||||
new Date().toISOString(),
|
||||
rows,
|
||||
new Date().toISOString(),
|
||||
);
|
||||
db.exec("COMMIT");
|
||||
// Old pages are reused by the next import; reclaiming them would
|
||||
// rewrite the whole file.
|
||||
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
||||
return rows;
|
||||
},
|
||||
abort() {
|
||||
if (db.isTransaction) db.exec("ROLLBACK");
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const state = (source) =>
|
||||
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
|
||||
function recordCheck(source, error = null) {
|
||||
db.prepare(
|
||||
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
|
||||
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
|
||||
).run(source, new Date().toISOString(), error);
|
||||
}
|
||||
const status = () =>
|
||||
db
|
||||
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
|
||||
.all();
|
||||
|
||||
return {
|
||||
db,
|
||||
lookup,
|
||||
remember,
|
||||
beginImport,
|
||||
state,
|
||||
recordCheck,
|
||||
status,
|
||||
close: () => db.close(),
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue