All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
175 lines
5.8 KiB
JavaScript
175 lines
5.8 KiB
JavaScript
import { createGunzip } from "node:zlib";
|
|
import { Readable } from "node:stream";
|
|
import { databaseGtin } from "../../shared/barcode.js";
|
|
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
|
|
|
|
export const OFF_CSV =
|
|
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
|
|
export const userAgent =
|
|
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
|
|
|
|
// Columns that are not nutrient amounts, or duplicate one handled separately.
|
|
const skipped = new Set([
|
|
"energy",
|
|
"energy-kj",
|
|
"energy-from-fat",
|
|
"salt",
|
|
"added-salt",
|
|
"ph",
|
|
"alcohol", // % vol in Open Food Facts, not grams
|
|
"cocoa",
|
|
"chlorophyl",
|
|
"collagen-meat-protein-ratio",
|
|
"glycemic-index",
|
|
"water-hardness",
|
|
"acidity",
|
|
"carbohydrates-total",
|
|
]);
|
|
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
|
|
|
|
/**
|
|
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
|
|
* stored units.
|
|
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
|
|
*/
|
|
export function offNutrients(entries) {
|
|
const raw = new Map();
|
|
for (const [key, value] of entries) {
|
|
const number = typeof value === "number" ? value : Number(value);
|
|
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
|
|
continue;
|
|
raw.set(key, number);
|
|
}
|
|
const nutrients = {};
|
|
let kcal = raw.get("energy-kcal");
|
|
if (kcal === undefined) {
|
|
const kj = raw.get("energy-kj") ?? raw.get("energy");
|
|
if (kj !== undefined) kcal = kj / 4.184;
|
|
}
|
|
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
|
|
let sodium = raw.get("sodium");
|
|
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
|
|
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
|
|
for (const [key, grams] of raw) {
|
|
if (
|
|
key === "energy-kcal" ||
|
|
key === "sodium" ||
|
|
skipped.has(key) ||
|
|
notNutrient.test(key) ||
|
|
!nutrientKeyPattern.test(key) ||
|
|
grams > 100 // more than 100 g per 100 g is a data-entry error
|
|
)
|
|
continue;
|
|
const value = fromGrams(key, grams);
|
|
if (value !== null) nutrients[key] = value;
|
|
}
|
|
return nutrients;
|
|
}
|
|
|
|
function servingUnit(text) {
|
|
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
|
|
? "ml"
|
|
: "g";
|
|
}
|
|
|
|
/** Shapes a CSV row or API product into a products-table row. */
|
|
export function offProduct({
|
|
code,
|
|
name,
|
|
brands,
|
|
serving,
|
|
servingQty,
|
|
nutrients,
|
|
}) {
|
|
const gtin = databaseGtin(code);
|
|
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
|
|
const qty = Number(servingQty);
|
|
return {
|
|
gtin,
|
|
name: (name || "").trim().slice(0, 200) || null,
|
|
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
|
|
serving: (serving || "").trim().slice(0, 100) || null,
|
|
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
|
|
servingUnit: servingUnit(serving),
|
|
nutrients,
|
|
};
|
|
}
|
|
|
|
async function* lines(stream) {
|
|
let rest = "";
|
|
stream.setEncoding("utf8");
|
|
for await (const chunk of stream) {
|
|
const parts = (rest + chunk).split("\n");
|
|
rest = parts.pop();
|
|
yield* parts;
|
|
}
|
|
if (rest) yield rest;
|
|
}
|
|
|
|
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
|
|
export async function importOffCsv(response, sink, { onProgress } = {}) {
|
|
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
|
|
let header;
|
|
let count = 0;
|
|
for await (const line of lines(gunzip)) {
|
|
const fields = line.split("\t");
|
|
if (!header) {
|
|
header = Object.fromEntries(fields.map((name, i) => [name, i]));
|
|
for (const column of ["code", "product_name", "energy-kcal_100g"])
|
|
if (!(column in header))
|
|
throw new Error(`Open Food Facts export is missing ${column}`);
|
|
header.nutrientColumns = fields
|
|
.map((name, i) => [name, i])
|
|
.filter(([name]) => name.endsWith("_100g"))
|
|
.map(([name, i]) => [name.slice(0, -5), i]);
|
|
continue;
|
|
}
|
|
if (fields[header.no_nutrition_data] === "on") continue;
|
|
const product = offProduct({
|
|
code: fields[header.code],
|
|
name: fields[header.product_name],
|
|
brands: fields[header.brands],
|
|
serving: fields[header.serving_size],
|
|
servingQty: fields[header.serving_quantity],
|
|
nutrients: offNutrients(
|
|
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
|
|
),
|
|
});
|
|
if (!product) continue;
|
|
sink.add(product);
|
|
if (++count % 250000 === 0) onProgress?.(count);
|
|
}
|
|
if (!header) throw new Error("Open Food Facts export was empty");
|
|
return count;
|
|
}
|
|
|
|
const apiFields =
|
|
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
|
|
/** Live lookup, used only when no local copy has the product. */
|
|
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
|
|
// Open Food Facts keys products by EAN-13 (or EAN-8).
|
|
const stripped = gtin.replace(/^0+/, "");
|
|
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
|
|
const response = await fetchImpl(
|
|
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
|
|
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
|
|
);
|
|
if (response.status === 404) return null;
|
|
if (!response.ok)
|
|
throw new Error(`Open Food Facts answered ${response.status}`);
|
|
const data = await response.json();
|
|
if (data.status !== 1 || !data.product) return null;
|
|
const p = data.product;
|
|
const entries = Object.entries(p.nutriments || {})
|
|
.filter(([key]) => key.endsWith("_100g"))
|
|
.map(([key, value]) => [key.slice(0, -5), value]);
|
|
const product = offProduct({
|
|
code: gtin,
|
|
name: p.product_name_en || p.product_name,
|
|
brands: p.brands,
|
|
serving: p.serving_size,
|
|
servingQty: p.serving_quantity,
|
|
nutrients: offNutrients(entries),
|
|
});
|
|
return product && { ...product, source: "api" };
|
|
}
|