Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f6981c8a2
commit
80fc48b9f1
39 changed files with 4434 additions and 33 deletions
398
server/products/usda.js
Normal file
398
server/products/usda.js
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
import { createWriteStream } from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import yauzl from "yauzl";
|
||||
import { databaseGtin } from "../../shared/barcode.js";
|
||||
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
|
||||
|
||||
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
|
||||
const archiveName =
|
||||
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
|
||||
|
||||
/** The newest Branded Foods CSV archive linked from the download page. */
|
||||
export async function latestUsdaArchive(fetchImpl = fetch) {
|
||||
const response = await fetchImpl(USDA_PAGE, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
|
||||
const versions = [...(await response.text()).matchAll(archiveName)].map(
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!versions.length) throw new Error("No USDA branded archive is listed");
|
||||
const version = versions.sort().at(-1);
|
||||
return {
|
||||
version,
|
||||
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function download(url, file, fetchImpl = fetch) {
|
||||
const response = await fetchImpl(url);
|
||||
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
|
||||
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
|
||||
}
|
||||
|
||||
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
|
||||
// are kept under a slug of their USDA name.
|
||||
const nutrientIds = {
|
||||
1008: "energy-kcal",
|
||||
1003: "proteins",
|
||||
1004: "fat",
|
||||
1005: "carbohydrates",
|
||||
1079: "fiber",
|
||||
2000: "sugars",
|
||||
1093: "sodium",
|
||||
1258: "saturated-fat",
|
||||
1257: "trans-fat",
|
||||
1292: "monounsaturated-fat",
|
||||
1293: "polyunsaturated-fat",
|
||||
1253: "cholesterol",
|
||||
1235: "added-sugars",
|
||||
1086: "polyols",
|
||||
1082: "soluble-fiber",
|
||||
1084: "insoluble-fiber",
|
||||
1009: "starch",
|
||||
1010: "sucrose",
|
||||
1011: "glucose",
|
||||
1012: "fructose",
|
||||
1013: "lactose",
|
||||
1014: "maltose",
|
||||
1075: "galactose",
|
||||
1056: "sorbitol",
|
||||
1078: "xylitol",
|
||||
1055: "mannitol",
|
||||
1051: "water",
|
||||
1018: "alcohol",
|
||||
1057: "caffeine",
|
||||
1234: "taurine",
|
||||
1068: "beta-glucan",
|
||||
1181: "inositol",
|
||||
1106: "vitamin-a",
|
||||
1104: "vitamin-a-iu",
|
||||
1162: "vitamin-c",
|
||||
1114: "vitamin-d",
|
||||
1109: "vitamin-e",
|
||||
1185: "vitamin-k",
|
||||
1165: "vitamin-b1",
|
||||
1166: "vitamin-b2",
|
||||
1167: "vitamin-pp",
|
||||
1175: "vitamin-b6",
|
||||
1190: "vitamin-b9",
|
||||
1177: "folates",
|
||||
1186: "folic-acid",
|
||||
1178: "vitamin-b12",
|
||||
1176: "biotin",
|
||||
1170: "pantothenic-acid",
|
||||
1180: "choline",
|
||||
1087: "calcium",
|
||||
1089: "iron",
|
||||
1092: "potassium",
|
||||
1090: "magnesium",
|
||||
1091: "phosphorus",
|
||||
1095: "zinc",
|
||||
1098: "copper",
|
||||
1101: "manganese",
|
||||
1103: "selenium",
|
||||
1100: "iodine",
|
||||
1096: "chromium",
|
||||
1102: "molybdenum",
|
||||
1088: "chloride",
|
||||
1099: "fluoride",
|
||||
};
|
||||
// Fallbacks used only when the preferred ID is absent.
|
||||
const fallbacks = {
|
||||
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
|
||||
1085: ["fat", (g) => g], // Total fat (NLEA)
|
||||
1063: ["sugars", (g) => g], // Sugars, Total
|
||||
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
|
||||
};
|
||||
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
|
||||
const slug = (name) =>
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 48);
|
||||
|
||||
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
|
||||
export function nutrientMapper(nutrientRows) {
|
||||
const map = new Map();
|
||||
for (const { id, name, unit } of nutrientRows) {
|
||||
const from = usdaUnits[unit];
|
||||
let key = nutrientIds[id];
|
||||
if (fallbacks[id]) {
|
||||
const [target, convert] = fallbacks[id];
|
||||
map.set(id, { key: target, convert, fallback: true });
|
||||
continue;
|
||||
}
|
||||
if (!key) {
|
||||
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
|
||||
continue;
|
||||
key = slug(name);
|
||||
if (!key) continue;
|
||||
}
|
||||
const to = nutrientUnit(key);
|
||||
const convert =
|
||||
from === to ? (value) => value : (value) => convertMass(value, from, to);
|
||||
if (from !== to && convertMass(1, from, to) === null) continue;
|
||||
map.set(id, { key, convert, fallback: false });
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Parses one record starting at `pos`; null when `text` ends mid-record.
|
||||
function parseRecord(text, pos) {
|
||||
const fields = [];
|
||||
for (;;) {
|
||||
if (pos >= text.length) return null;
|
||||
if (text[pos] === '"') {
|
||||
let value = "";
|
||||
let i = pos + 1;
|
||||
for (;;) {
|
||||
const quote = text.indexOf('"', i);
|
||||
if (quote === -1 || quote + 1 >= text.length) return null;
|
||||
value += text.slice(i, quote);
|
||||
if (text[quote + 1] !== '"') {
|
||||
pos = quote + 1;
|
||||
break;
|
||||
}
|
||||
value += '"'; // "" inside a quoted field
|
||||
i = quote + 2;
|
||||
}
|
||||
fields.push(value);
|
||||
} else {
|
||||
let end = pos;
|
||||
while (end < text.length && text[end] !== "," && text[end] !== "\n")
|
||||
end++;
|
||||
if (end >= text.length) return null;
|
||||
fields.push(text.slice(pos, end).replace(/\r$/, ""));
|
||||
pos = end;
|
||||
}
|
||||
const c = text[pos];
|
||||
if (c === ",") pos++;
|
||||
else if (c === "\n") return { fields, next: pos + 1 };
|
||||
else if (c === "\r" && text[pos + 1] === "\n")
|
||||
return { fields, next: pos + 2 };
|
||||
else if (pos + 1 >= text.length) return null;
|
||||
else pos++; // stray character after a closing quote
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal RFC 4180 reader: yields one array of fields per record. */
|
||||
export async function* csvRecords(stream) {
|
||||
stream.setEncoding("utf8");
|
||||
let buffer = "";
|
||||
for await (const chunk of stream) {
|
||||
buffer += chunk;
|
||||
let start = 0;
|
||||
for (let record; (record = parseRecord(buffer, start)); start = record.next)
|
||||
yield record.fields;
|
||||
buffer = buffer.slice(start);
|
||||
}
|
||||
if (buffer.trim()) {
|
||||
const record = parseRecord(buffer + "\n", 0);
|
||||
if (record) yield record.fields;
|
||||
}
|
||||
}
|
||||
|
||||
function openZip(file) {
|
||||
return new Promise((resolve, reject) =>
|
||||
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
|
||||
if (error) return reject(error);
|
||||
const entries = new Map();
|
||||
zip.on("entry", (entry) =>
|
||||
entries.set(entry.fileName.split("/").at(-1), entry),
|
||||
);
|
||||
zip.on("end", () => resolve({ zip, entries }));
|
||||
zip.on("error", reject);
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Yields the requested columns of each row, in the order asked for.
|
||||
async function* table(zip, entries, name, columns) {
|
||||
const entry = entries.get(name);
|
||||
if (!entry) throw new Error(`USDA archive is missing ${name}`);
|
||||
const stream = await new Promise((resolve, reject) =>
|
||||
zip.openReadStream(entry, (error, s) =>
|
||||
error ? reject(error) : resolve(s),
|
||||
),
|
||||
);
|
||||
let indexes;
|
||||
for await (const fields of csvRecords(stream)) {
|
||||
if (!indexes) {
|
||||
indexes = columns.map((column) => fields.indexOf(column));
|
||||
const missing = columns.filter((_, i) => indexes[i] < 0);
|
||||
if (missing.length)
|
||||
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
|
||||
continue;
|
||||
}
|
||||
yield indexes.map((i) => fields[i]);
|
||||
}
|
||||
}
|
||||
|
||||
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
|
||||
function servingText(household, qty, unit) {
|
||||
const amount = `${+qty.toFixed(1)} ${unit}`;
|
||||
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
|
||||
return amount;
|
||||
// Some submissions already include the weight: "3/4 cup (20g)".
|
||||
return /\d\s*(g|ml|grams?)\b/i.test(household)
|
||||
? household
|
||||
: `${household} (${amount})`;
|
||||
}
|
||||
|
||||
/** Reads a Branded Foods CSV archive into `sink`. */
|
||||
export async function importUsdaZip(file, sink, { onProgress } = {}) {
|
||||
const { zip, entries } = await openZip(file);
|
||||
try {
|
||||
const nutrientRows = [];
|
||||
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
|
||||
"id",
|
||||
"name",
|
||||
"unit_name",
|
||||
]))
|
||||
nutrientRows.push({ id: Number(id), name, unit });
|
||||
const mapper = nutrientMapper(nutrientRows);
|
||||
|
||||
// A GTIN can appear several times as a product is resubmitted; keep the
|
||||
// newest submission.
|
||||
const byGtin = new Map();
|
||||
const strings = new Map();
|
||||
// Substrings keep their whole parse buffer alive; store fresh copies,
|
||||
// shared across the many products with the same brand or serving.
|
||||
const keep = (text) => {
|
||||
if (!text) return "";
|
||||
let copy = strings.get(text);
|
||||
if (copy === undefined) {
|
||||
copy = Buffer.from(text).toString();
|
||||
strings.set(copy, copy);
|
||||
}
|
||||
return copy;
|
||||
};
|
||||
for await (const [
|
||||
id,
|
||||
gtinUpc,
|
||||
brandName,
|
||||
brandOwner,
|
||||
servingSize,
|
||||
servingSizeUnit,
|
||||
householdText,
|
||||
available,
|
||||
modified,
|
||||
] of table(zip, entries, "branded_food.csv", [
|
||||
"fdc_id",
|
||||
"gtin_upc",
|
||||
"brand_name",
|
||||
"brand_owner",
|
||||
"serving_size",
|
||||
"serving_size_unit",
|
||||
"household_serving_fulltext",
|
||||
"available_date",
|
||||
"modified_date",
|
||||
])) {
|
||||
const gtin = databaseGtin(gtinUpc);
|
||||
if (!gtin) continue;
|
||||
const date = available || modified || "";
|
||||
const fdcId = Number(id);
|
||||
const current = byGtin.get(gtin);
|
||||
if (
|
||||
current &&
|
||||
(current.date > date ||
|
||||
(current.date === date && current.fdcId > fdcId))
|
||||
)
|
||||
continue;
|
||||
const qty = Number(servingSize);
|
||||
const household = (householdText || "").trim();
|
||||
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
|
||||
byGtin.set(keep(gtin), {
|
||||
fdcId,
|
||||
date: keep(date),
|
||||
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
|
||||
servingQty: unit && qty > 0 ? qty : null,
|
||||
servingUnit: unit || "g",
|
||||
serving:
|
||||
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
|
||||
name: "",
|
||||
gtin: "",
|
||||
});
|
||||
}
|
||||
strings.clear();
|
||||
const byFdc = new Map();
|
||||
for (const [gtin, product] of byGtin) {
|
||||
product.gtin = gtin;
|
||||
byFdc.set(product.fdcId, product);
|
||||
}
|
||||
byGtin.clear();
|
||||
|
||||
for await (const [id, description] of table(zip, entries, "food.csv", [
|
||||
"fdc_id",
|
||||
"description",
|
||||
])) {
|
||||
const product = byFdc.get(Number(id));
|
||||
if (product)
|
||||
product.name = Buffer.from(description.slice(0, 200)).toString();
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
const finish = (product, values) => {
|
||||
const nutrients = {};
|
||||
const late = [];
|
||||
for (let i = 0; i < values.length; i += 2) {
|
||||
const mapping = mapper.get(values[i]);
|
||||
const number = values[i + 1];
|
||||
if (!mapping || !Number.isFinite(number) || number < 0) continue;
|
||||
if (mapping.fallback) late.push([mapping, number]);
|
||||
else {
|
||||
const value = mapping.convert(number);
|
||||
if (value !== null) nutrients[mapping.key] = value;
|
||||
}
|
||||
}
|
||||
for (const [mapping, number] of late)
|
||||
if (nutrients[mapping.key] === undefined)
|
||||
nutrients[mapping.key] = mapping.convert(number);
|
||||
if (
|
||||
nutrients["energy-kcal"] === undefined ||
|
||||
nutrients["energy-kcal"] > 950
|
||||
)
|
||||
return;
|
||||
sink.add({
|
||||
gtin: product.gtin,
|
||||
name: product.name,
|
||||
brand: product.brand,
|
||||
serving: product.serving,
|
||||
servingQty: product.servingQty,
|
||||
servingUnit: product.servingUnit,
|
||||
nutrients,
|
||||
});
|
||||
if (++count % 250000 === 0) onProgress?.(count);
|
||||
};
|
||||
// ~30 million rows, grouped by food: each product is finished as soon as
|
||||
// the next one starts, so only one food's nutrients are held at a time.
|
||||
let currentId = null;
|
||||
let values = [];
|
||||
for await (const [id, nutrientId, amount] of table(
|
||||
zip,
|
||||
entries,
|
||||
"food_nutrient.csv",
|
||||
["fdc_id", "nutrient_id", "amount"],
|
||||
)) {
|
||||
const fdcId = Number(id);
|
||||
if (fdcId !== currentId) {
|
||||
const done = byFdc.get(currentId);
|
||||
if (done) {
|
||||
finish(done, values);
|
||||
byFdc.delete(currentId);
|
||||
}
|
||||
currentId = fdcId;
|
||||
values = [];
|
||||
}
|
||||
if (byFdc.has(fdcId) && amount !== "")
|
||||
values.push(Number(nutrientId), Number(amount));
|
||||
}
|
||||
const last = byFdc.get(currentId);
|
||||
if (last) finish(last, values);
|
||||
return count;
|
||||
} finally {
|
||||
zip.close();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue