Calio/server/products/db.js
Elijah 80fc48b9f1
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Add barcode and nutrition label scanning
Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 13:41:44 -07:00

165 lines
5.3 KiB
JavaScript

import { DatabaseSync } from "node:sqlite";
import { titleCase } from "../../shared/products.js";
// Each source lives in its own table so a refresh can be built alongside the
// old copy and swapped in atomically. The file is a cache: it can be deleted
// at any time and is rebuilt on the next import.
export const sources = ["usda", "off", "api"];
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
export function openProducts(filename) {
const db = new DatabaseSync(filename);
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
for (const source of ["usda", "off"])
db.exec(
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
);
// API answers include misses (nutrients NULL) so repeated scans of an
// unknown code don't hit Open Food Facts again.
db.exec(
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
);
const select = Object.fromEntries(
sources.map((source) => [
source,
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
]),
);
const toProduct = (row, source) =>
row?.nutrients
? {
gtin: row.gtin,
name: source === "usda" ? titleCase(row.name) : row.name,
brand: source === "usda" ? titleCase(row.brand) : row.brand,
serving: row.serving,
servingQty: row.serving_qty,
servingUnit: row.serving_unit,
nutrients: JSON.parse(row.nutrients),
source,
}
: null;
/** The best local match, or `undefined` when no source has an answer. */
function lookup(
gtin,
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
) {
const found = {};
for (const source of sources) found[source] = select[source].get(gtin);
const api = found.api;
if (
api &&
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
)
found.api = undefined;
const usda = toProduct(found.usda, "usda");
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
// USDA comes from manufacturers' own label submissions, so it wins;
// community-entered Open Food Facts fills in what it lacks.
const base = usda || off;
if (base)
return {
...base,
name: base.name || off?.name,
brand: base.brand || off?.brand,
};
return found.api ? null : undefined;
}
const upsertApi = db.prepare(
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
);
function remember(gtin, product) {
upsertApi.run(
gtin,
product?.name ?? null,
product?.brand ?? null,
product?.serving ?? null,
product?.servingQty ?? null,
product?.servingUnit ?? null,
product ? JSON.stringify(product.nutrients) : null,
Date.now(),
);
}
/** Bulk loader used by the importers: writes `<source>_products_next`. */
function beginImport(source) {
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
const insert = db.prepare(
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
);
let pending = 0;
let rows = 0;
db.exec("BEGIN");
return {
add(product) {
insert.run(
product.gtin,
product.name || null,
product.brand || null,
product.serving || null,
product.servingQty ?? null,
product.servingUnit || null,
JSON.stringify(product.nutrients),
);
rows++;
if (++pending >= 20000) {
db.exec("COMMIT; BEGIN");
pending = 0;
}
},
commit(version) {
db.exec(`COMMIT; BEGIN IMMEDIATE;
DROP TABLE IF EXISTS ${source}_products;
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
db.prepare(
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
VALUES (?,?,?,?,?,NULL)`,
).run(
source,
version,
new Date().toISOString(),
rows,
new Date().toISOString(),
);
db.exec("COMMIT");
// Old pages are reused by the next import; reclaiming them would
// rewrite the whole file.
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
return rows;
},
abort() {
if (db.isTransaction) db.exec("ROLLBACK");
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
},
};
}
const state = (source) =>
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
function recordCheck(source, error = null) {
db.prepare(
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
).run(source, new Date().toISOString(), error);
}
const status = () =>
db
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
.all();
return {
db,
lookup,
remember,
beginImport,
state,
recordCheck,
status,
close: () => db.close(),
};
}