Calio/server/products/off.js
Elijah 80fc48b9f1
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Add barcode and nutrition label scanning
Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 13:41:44 -07:00

175 lines
5.8 KiB
JavaScript

import { createGunzip } from "node:zlib";
import { Readable } from "node:stream";
import { databaseGtin } from "../../shared/barcode.js";
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
export const OFF_CSV =
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
export const userAgent =
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
// Columns that are not nutrient amounts, or duplicate one handled separately.
const skipped = new Set([
"energy",
"energy-kj",
"energy-from-fat",
"salt",
"added-salt",
"ph",
"alcohol", // % vol in Open Food Facts, not grams
"cocoa",
"chlorophyl",
"collagen-meat-protein-ratio",
"glycemic-index",
"water-hardness",
"acidity",
"carbohydrates-total",
]);
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
/**
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
* stored units.
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
*/
export function offNutrients(entries) {
const raw = new Map();
for (const [key, value] of entries) {
const number = typeof value === "number" ? value : Number(value);
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
continue;
raw.set(key, number);
}
const nutrients = {};
let kcal = raw.get("energy-kcal");
if (kcal === undefined) {
const kj = raw.get("energy-kj") ?? raw.get("energy");
if (kj !== undefined) kcal = kj / 4.184;
}
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
let sodium = raw.get("sodium");
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
for (const [key, grams] of raw) {
if (
key === "energy-kcal" ||
key === "sodium" ||
skipped.has(key) ||
notNutrient.test(key) ||
!nutrientKeyPattern.test(key) ||
grams > 100 // more than 100 g per 100 g is a data-entry error
)
continue;
const value = fromGrams(key, grams);
if (value !== null) nutrients[key] = value;
}
return nutrients;
}
function servingUnit(text) {
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
? "ml"
: "g";
}
/** Shapes a CSV row or API product into a products-table row. */
export function offProduct({
code,
name,
brands,
serving,
servingQty,
nutrients,
}) {
const gtin = databaseGtin(code);
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
const qty = Number(servingQty);
return {
gtin,
name: (name || "").trim().slice(0, 200) || null,
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
serving: (serving || "").trim().slice(0, 100) || null,
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
servingUnit: servingUnit(serving),
nutrients,
};
}
async function* lines(stream) {
let rest = "";
stream.setEncoding("utf8");
for await (const chunk of stream) {
const parts = (rest + chunk).split("\n");
rest = parts.pop();
yield* parts;
}
if (rest) yield rest;
}
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
export async function importOffCsv(response, sink, { onProgress } = {}) {
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
let header;
let count = 0;
for await (const line of lines(gunzip)) {
const fields = line.split("\t");
if (!header) {
header = Object.fromEntries(fields.map((name, i) => [name, i]));
for (const column of ["code", "product_name", "energy-kcal_100g"])
if (!(column in header))
throw new Error(`Open Food Facts export is missing ${column}`);
header.nutrientColumns = fields
.map((name, i) => [name, i])
.filter(([name]) => name.endsWith("_100g"))
.map(([name, i]) => [name.slice(0, -5), i]);
continue;
}
if (fields[header.no_nutrition_data] === "on") continue;
const product = offProduct({
code: fields[header.code],
name: fields[header.product_name],
brands: fields[header.brands],
serving: fields[header.serving_size],
servingQty: fields[header.serving_quantity],
nutrients: offNutrients(
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
),
});
if (!product) continue;
sink.add(product);
if (++count % 250000 === 0) onProgress?.(count);
}
if (!header) throw new Error("Open Food Facts export was empty");
return count;
}
const apiFields =
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
/** Live lookup, used only when no local copy has the product. */
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
// Open Food Facts keys products by EAN-13 (or EAN-8).
const stripped = gtin.replace(/^0+/, "");
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
const response = await fetchImpl(
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
);
if (response.status === 404) return null;
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const data = await response.json();
if (data.status !== 1 || !data.product) return null;
const p = data.product;
const entries = Object.entries(p.nutriments || {})
.filter(([key]) => key.endsWith("_100g"))
.map(([key, value]) => [key.slice(0, -5), value]);
const product = offProduct({
code: gtin,
name: p.product_name_en || p.product_name,
brands: p.brands,
serving: p.serving_size,
servingQty: p.serving_quantity,
nutrients: offNutrients(entries),
});
return product && { ...product, source: "api" };
}