Calio/server/products/usda.js
Elijah 80fc48b9f1
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Add barcode and nutrition label scanning
Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 13:41:44 -07:00

398 lines
12 KiB
JavaScript

import { createWriteStream } from "node:fs";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import yauzl from "yauzl";
import { databaseGtin } from "../../shared/barcode.js";
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
const archiveName =
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
/** The newest Branded Foods CSV archive linked from the download page. */
export async function latestUsdaArchive(fetchImpl = fetch) {
const response = await fetchImpl(USDA_PAGE, {
signal: AbortSignal.timeout(30000),
});
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
const versions = [...(await response.text()).matchAll(archiveName)].map(
(match) => match[1],
);
if (!versions.length) throw new Error("No USDA branded archive is listed");
const version = versions.sort().at(-1);
return {
version,
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
};
}
export async function download(url, file, fetchImpl = fetch) {
const response = await fetchImpl(url);
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
}
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
// are kept under a slug of their USDA name.
const nutrientIds = {
1008: "energy-kcal",
1003: "proteins",
1004: "fat",
1005: "carbohydrates",
1079: "fiber",
2000: "sugars",
1093: "sodium",
1258: "saturated-fat",
1257: "trans-fat",
1292: "monounsaturated-fat",
1293: "polyunsaturated-fat",
1253: "cholesterol",
1235: "added-sugars",
1086: "polyols",
1082: "soluble-fiber",
1084: "insoluble-fiber",
1009: "starch",
1010: "sucrose",
1011: "glucose",
1012: "fructose",
1013: "lactose",
1014: "maltose",
1075: "galactose",
1056: "sorbitol",
1078: "xylitol",
1055: "mannitol",
1051: "water",
1018: "alcohol",
1057: "caffeine",
1234: "taurine",
1068: "beta-glucan",
1181: "inositol",
1106: "vitamin-a",
1104: "vitamin-a-iu",
1162: "vitamin-c",
1114: "vitamin-d",
1109: "vitamin-e",
1185: "vitamin-k",
1165: "vitamin-b1",
1166: "vitamin-b2",
1167: "vitamin-pp",
1175: "vitamin-b6",
1190: "vitamin-b9",
1177: "folates",
1186: "folic-acid",
1178: "vitamin-b12",
1176: "biotin",
1170: "pantothenic-acid",
1180: "choline",
1087: "calcium",
1089: "iron",
1092: "potassium",
1090: "magnesium",
1091: "phosphorus",
1095: "zinc",
1098: "copper",
1101: "manganese",
1103: "selenium",
1100: "iodine",
1096: "chromium",
1102: "molybdenum",
1088: "chloride",
1099: "fluoride",
};
// Fallbacks used only when the preferred ID is absent.
const fallbacks = {
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
1085: ["fat", (g) => g], // Total fat (NLEA)
1063: ["sugars", (g) => g], // Sugars, Total
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
};
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
const slug = (name) =>
name
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, "")
.slice(0, 48);
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
export function nutrientMapper(nutrientRows) {
const map = new Map();
for (const { id, name, unit } of nutrientRows) {
const from = usdaUnits[unit];
let key = nutrientIds[id];
if (fallbacks[id]) {
const [target, convert] = fallbacks[id];
map.set(id, { key: target, convert, fallback: true });
continue;
}
if (!key) {
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
continue;
key = slug(name);
if (!key) continue;
}
const to = nutrientUnit(key);
const convert =
from === to ? (value) => value : (value) => convertMass(value, from, to);
if (from !== to && convertMass(1, from, to) === null) continue;
map.set(id, { key, convert, fallback: false });
}
return map;
}
// Parses one record starting at `pos`; null when `text` ends mid-record.
function parseRecord(text, pos) {
const fields = [];
for (;;) {
if (pos >= text.length) return null;
if (text[pos] === '"') {
let value = "";
let i = pos + 1;
for (;;) {
const quote = text.indexOf('"', i);
if (quote === -1 || quote + 1 >= text.length) return null;
value += text.slice(i, quote);
if (text[quote + 1] !== '"') {
pos = quote + 1;
break;
}
value += '"'; // "" inside a quoted field
i = quote + 2;
}
fields.push(value);
} else {
let end = pos;
while (end < text.length && text[end] !== "," && text[end] !== "\n")
end++;
if (end >= text.length) return null;
fields.push(text.slice(pos, end).replace(/\r$/, ""));
pos = end;
}
const c = text[pos];
if (c === ",") pos++;
else if (c === "\n") return { fields, next: pos + 1 };
else if (c === "\r" && text[pos + 1] === "\n")
return { fields, next: pos + 2 };
else if (pos + 1 >= text.length) return null;
else pos++; // stray character after a closing quote
}
}
/** Minimal RFC 4180 reader: yields one array of fields per record. */
export async function* csvRecords(stream) {
stream.setEncoding("utf8");
let buffer = "";
for await (const chunk of stream) {
buffer += chunk;
let start = 0;
for (let record; (record = parseRecord(buffer, start)); start = record.next)
yield record.fields;
buffer = buffer.slice(start);
}
if (buffer.trim()) {
const record = parseRecord(buffer + "\n", 0);
if (record) yield record.fields;
}
}
function openZip(file) {
return new Promise((resolve, reject) =>
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
if (error) return reject(error);
const entries = new Map();
zip.on("entry", (entry) =>
entries.set(entry.fileName.split("/").at(-1), entry),
);
zip.on("end", () => resolve({ zip, entries }));
zip.on("error", reject);
}),
);
}
// Yields the requested columns of each row, in the order asked for.
async function* table(zip, entries, name, columns) {
const entry = entries.get(name);
if (!entry) throw new Error(`USDA archive is missing ${name}`);
const stream = await new Promise((resolve, reject) =>
zip.openReadStream(entry, (error, s) =>
error ? reject(error) : resolve(s),
),
);
let indexes;
for await (const fields of csvRecords(stream)) {
if (!indexes) {
indexes = columns.map((column) => fields.indexOf(column));
const missing = columns.filter((_, i) => indexes[i] < 0);
if (missing.length)
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
continue;
}
yield indexes.map((i) => fields[i]);
}
}
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
function servingText(household, qty, unit) {
const amount = `${+qty.toFixed(1)} ${unit}`;
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
return amount;
// Some submissions already include the weight: "3/4 cup (20g)".
return /\d\s*(g|ml|grams?)\b/i.test(household)
? household
: `${household} (${amount})`;
}
/** Reads a Branded Foods CSV archive into `sink`. */
export async function importUsdaZip(file, sink, { onProgress } = {}) {
const { zip, entries } = await openZip(file);
try {
const nutrientRows = [];
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
"id",
"name",
"unit_name",
]))
nutrientRows.push({ id: Number(id), name, unit });
const mapper = nutrientMapper(nutrientRows);
// A GTIN can appear several times as a product is resubmitted; keep the
// newest submission.
const byGtin = new Map();
const strings = new Map();
// Substrings keep their whole parse buffer alive; store fresh copies,
// shared across the many products with the same brand or serving.
const keep = (text) => {
if (!text) return "";
let copy = strings.get(text);
if (copy === undefined) {
copy = Buffer.from(text).toString();
strings.set(copy, copy);
}
return copy;
};
for await (const [
id,
gtinUpc,
brandName,
brandOwner,
servingSize,
servingSizeUnit,
householdText,
available,
modified,
] of table(zip, entries, "branded_food.csv", [
"fdc_id",
"gtin_upc",
"brand_name",
"brand_owner",
"serving_size",
"serving_size_unit",
"household_serving_fulltext",
"available_date",
"modified_date",
])) {
const gtin = databaseGtin(gtinUpc);
if (!gtin) continue;
const date = available || modified || "";
const fdcId = Number(id);
const current = byGtin.get(gtin);
if (
current &&
(current.date > date ||
(current.date === date && current.fdcId > fdcId))
)
continue;
const qty = Number(servingSize);
const household = (householdText || "").trim();
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
byGtin.set(keep(gtin), {
fdcId,
date: keep(date),
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
servingQty: unit && qty > 0 ? qty : null,
servingUnit: unit || "g",
serving:
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
name: "",
gtin: "",
});
}
strings.clear();
const byFdc = new Map();
for (const [gtin, product] of byGtin) {
product.gtin = gtin;
byFdc.set(product.fdcId, product);
}
byGtin.clear();
for await (const [id, description] of table(zip, entries, "food.csv", [
"fdc_id",
"description",
])) {
const product = byFdc.get(Number(id));
if (product)
product.name = Buffer.from(description.slice(0, 200)).toString();
}
let count = 0;
const finish = (product, values) => {
const nutrients = {};
const late = [];
for (let i = 0; i < values.length; i += 2) {
const mapping = mapper.get(values[i]);
const number = values[i + 1];
if (!mapping || !Number.isFinite(number) || number < 0) continue;
if (mapping.fallback) late.push([mapping, number]);
else {
const value = mapping.convert(number);
if (value !== null) nutrients[mapping.key] = value;
}
}
for (const [mapping, number] of late)
if (nutrients[mapping.key] === undefined)
nutrients[mapping.key] = mapping.convert(number);
if (
nutrients["energy-kcal"] === undefined ||
nutrients["energy-kcal"] > 950
)
return;
sink.add({
gtin: product.gtin,
name: product.name,
brand: product.brand,
serving: product.serving,
servingQty: product.servingQty,
servingUnit: product.servingUnit,
nutrients,
});
if (++count % 250000 === 0) onProgress?.(count);
};
// ~30 million rows, grouped by food: each product is finished as soon as
// the next one starts, so only one food's nutrients are held at a time.
let currentId = null;
let values = [];
for await (const [id, nutrientId, amount] of table(
zip,
entries,
"food_nutrient.csv",
["fdc_id", "nutrient_id", "amount"],
)) {
const fdcId = Number(id);
if (fdcId !== currentId) {
const done = byFdc.get(currentId);
if (done) {
finish(done, values);
byFdc.delete(currentId);
}
currentId = fdcId;
values = [];
}
if (byFdc.has(fdcId) && amount !== "")
values.push(Number(nutrientId), Number(amount));
}
const last = byFdc.get(currentId);
if (last) finish(last, values);
return count;
} finally {
zip.close();
}
}