All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
398 lines
12 KiB
JavaScript
398 lines
12 KiB
JavaScript
import { createWriteStream } from "node:fs";
|
|
import { Readable } from "node:stream";
|
|
import { pipeline } from "node:stream/promises";
|
|
import yauzl from "yauzl";
|
|
import { databaseGtin } from "../../shared/barcode.js";
|
|
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
|
|
|
|
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
|
|
const archiveName =
|
|
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
|
|
|
|
/** The newest Branded Foods CSV archive linked from the download page. */
|
|
export async function latestUsdaArchive(fetchImpl = fetch) {
|
|
const response = await fetchImpl(USDA_PAGE, {
|
|
signal: AbortSignal.timeout(30000),
|
|
});
|
|
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
|
|
const versions = [...(await response.text()).matchAll(archiveName)].map(
|
|
(match) => match[1],
|
|
);
|
|
if (!versions.length) throw new Error("No USDA branded archive is listed");
|
|
const version = versions.sort().at(-1);
|
|
return {
|
|
version,
|
|
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
|
|
};
|
|
}
|
|
|
|
export async function download(url, file, fetchImpl = fetch) {
|
|
const response = await fetchImpl(url);
|
|
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
|
|
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
|
|
}
|
|
|
|
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
|
|
// are kept under a slug of their USDA name.
|
|
const nutrientIds = {
|
|
1008: "energy-kcal",
|
|
1003: "proteins",
|
|
1004: "fat",
|
|
1005: "carbohydrates",
|
|
1079: "fiber",
|
|
2000: "sugars",
|
|
1093: "sodium",
|
|
1258: "saturated-fat",
|
|
1257: "trans-fat",
|
|
1292: "monounsaturated-fat",
|
|
1293: "polyunsaturated-fat",
|
|
1253: "cholesterol",
|
|
1235: "added-sugars",
|
|
1086: "polyols",
|
|
1082: "soluble-fiber",
|
|
1084: "insoluble-fiber",
|
|
1009: "starch",
|
|
1010: "sucrose",
|
|
1011: "glucose",
|
|
1012: "fructose",
|
|
1013: "lactose",
|
|
1014: "maltose",
|
|
1075: "galactose",
|
|
1056: "sorbitol",
|
|
1078: "xylitol",
|
|
1055: "mannitol",
|
|
1051: "water",
|
|
1018: "alcohol",
|
|
1057: "caffeine",
|
|
1234: "taurine",
|
|
1068: "beta-glucan",
|
|
1181: "inositol",
|
|
1106: "vitamin-a",
|
|
1104: "vitamin-a-iu",
|
|
1162: "vitamin-c",
|
|
1114: "vitamin-d",
|
|
1109: "vitamin-e",
|
|
1185: "vitamin-k",
|
|
1165: "vitamin-b1",
|
|
1166: "vitamin-b2",
|
|
1167: "vitamin-pp",
|
|
1175: "vitamin-b6",
|
|
1190: "vitamin-b9",
|
|
1177: "folates",
|
|
1186: "folic-acid",
|
|
1178: "vitamin-b12",
|
|
1176: "biotin",
|
|
1170: "pantothenic-acid",
|
|
1180: "choline",
|
|
1087: "calcium",
|
|
1089: "iron",
|
|
1092: "potassium",
|
|
1090: "magnesium",
|
|
1091: "phosphorus",
|
|
1095: "zinc",
|
|
1098: "copper",
|
|
1101: "manganese",
|
|
1103: "selenium",
|
|
1100: "iodine",
|
|
1096: "chromium",
|
|
1102: "molybdenum",
|
|
1088: "chloride",
|
|
1099: "fluoride",
|
|
};
|
|
// Fallbacks used only when the preferred ID is absent.
|
|
const fallbacks = {
|
|
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
|
|
1085: ["fat", (g) => g], // Total fat (NLEA)
|
|
1063: ["sugars", (g) => g], // Sugars, Total
|
|
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
|
|
};
|
|
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
|
|
const slug = (name) =>
|
|
name
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9]+/g, "-")
|
|
.replace(/^-|-$/g, "")
|
|
.slice(0, 48);
|
|
|
|
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
|
|
export function nutrientMapper(nutrientRows) {
|
|
const map = new Map();
|
|
for (const { id, name, unit } of nutrientRows) {
|
|
const from = usdaUnits[unit];
|
|
let key = nutrientIds[id];
|
|
if (fallbacks[id]) {
|
|
const [target, convert] = fallbacks[id];
|
|
map.set(id, { key: target, convert, fallback: true });
|
|
continue;
|
|
}
|
|
if (!key) {
|
|
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
|
|
continue;
|
|
key = slug(name);
|
|
if (!key) continue;
|
|
}
|
|
const to = nutrientUnit(key);
|
|
const convert =
|
|
from === to ? (value) => value : (value) => convertMass(value, from, to);
|
|
if (from !== to && convertMass(1, from, to) === null) continue;
|
|
map.set(id, { key, convert, fallback: false });
|
|
}
|
|
return map;
|
|
}
|
|
|
|
// Parses one record starting at `pos`; null when `text` ends mid-record.
|
|
function parseRecord(text, pos) {
|
|
const fields = [];
|
|
for (;;) {
|
|
if (pos >= text.length) return null;
|
|
if (text[pos] === '"') {
|
|
let value = "";
|
|
let i = pos + 1;
|
|
for (;;) {
|
|
const quote = text.indexOf('"', i);
|
|
if (quote === -1 || quote + 1 >= text.length) return null;
|
|
value += text.slice(i, quote);
|
|
if (text[quote + 1] !== '"') {
|
|
pos = quote + 1;
|
|
break;
|
|
}
|
|
value += '"'; // "" inside a quoted field
|
|
i = quote + 2;
|
|
}
|
|
fields.push(value);
|
|
} else {
|
|
let end = pos;
|
|
while (end < text.length && text[end] !== "," && text[end] !== "\n")
|
|
end++;
|
|
if (end >= text.length) return null;
|
|
fields.push(text.slice(pos, end).replace(/\r$/, ""));
|
|
pos = end;
|
|
}
|
|
const c = text[pos];
|
|
if (c === ",") pos++;
|
|
else if (c === "\n") return { fields, next: pos + 1 };
|
|
else if (c === "\r" && text[pos + 1] === "\n")
|
|
return { fields, next: pos + 2 };
|
|
else if (pos + 1 >= text.length) return null;
|
|
else pos++; // stray character after a closing quote
|
|
}
|
|
}
|
|
|
|
/** Minimal RFC 4180 reader: yields one array of fields per record. */
|
|
export async function* csvRecords(stream) {
|
|
stream.setEncoding("utf8");
|
|
let buffer = "";
|
|
for await (const chunk of stream) {
|
|
buffer += chunk;
|
|
let start = 0;
|
|
for (let record; (record = parseRecord(buffer, start)); start = record.next)
|
|
yield record.fields;
|
|
buffer = buffer.slice(start);
|
|
}
|
|
if (buffer.trim()) {
|
|
const record = parseRecord(buffer + "\n", 0);
|
|
if (record) yield record.fields;
|
|
}
|
|
}
|
|
|
|
function openZip(file) {
|
|
return new Promise((resolve, reject) =>
|
|
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
|
|
if (error) return reject(error);
|
|
const entries = new Map();
|
|
zip.on("entry", (entry) =>
|
|
entries.set(entry.fileName.split("/").at(-1), entry),
|
|
);
|
|
zip.on("end", () => resolve({ zip, entries }));
|
|
zip.on("error", reject);
|
|
}),
|
|
);
|
|
}
|
|
// Yields the requested columns of each row, in the order asked for.
|
|
async function* table(zip, entries, name, columns) {
|
|
const entry = entries.get(name);
|
|
if (!entry) throw new Error(`USDA archive is missing ${name}`);
|
|
const stream = await new Promise((resolve, reject) =>
|
|
zip.openReadStream(entry, (error, s) =>
|
|
error ? reject(error) : resolve(s),
|
|
),
|
|
);
|
|
let indexes;
|
|
for await (const fields of csvRecords(stream)) {
|
|
if (!indexes) {
|
|
indexes = columns.map((column) => fields.indexOf(column));
|
|
const missing = columns.filter((_, i) => indexes[i] < 0);
|
|
if (missing.length)
|
|
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
|
|
continue;
|
|
}
|
|
yield indexes.map((i) => fields[i]);
|
|
}
|
|
}
|
|
|
|
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
|
|
function servingText(household, qty, unit) {
|
|
const amount = `${+qty.toFixed(1)} ${unit}`;
|
|
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
|
|
return amount;
|
|
// Some submissions already include the weight: "3/4 cup (20g)".
|
|
return /\d\s*(g|ml|grams?)\b/i.test(household)
|
|
? household
|
|
: `${household} (${amount})`;
|
|
}
|
|
|
|
/** Reads a Branded Foods CSV archive into `sink`. */
|
|
export async function importUsdaZip(file, sink, { onProgress } = {}) {
|
|
const { zip, entries } = await openZip(file);
|
|
try {
|
|
const nutrientRows = [];
|
|
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
|
|
"id",
|
|
"name",
|
|
"unit_name",
|
|
]))
|
|
nutrientRows.push({ id: Number(id), name, unit });
|
|
const mapper = nutrientMapper(nutrientRows);
|
|
|
|
// A GTIN can appear several times as a product is resubmitted; keep the
|
|
// newest submission.
|
|
const byGtin = new Map();
|
|
const strings = new Map();
|
|
// Substrings keep their whole parse buffer alive; store fresh copies,
|
|
// shared across the many products with the same brand or serving.
|
|
const keep = (text) => {
|
|
if (!text) return "";
|
|
let copy = strings.get(text);
|
|
if (copy === undefined) {
|
|
copy = Buffer.from(text).toString();
|
|
strings.set(copy, copy);
|
|
}
|
|
return copy;
|
|
};
|
|
for await (const [
|
|
id,
|
|
gtinUpc,
|
|
brandName,
|
|
brandOwner,
|
|
servingSize,
|
|
servingSizeUnit,
|
|
householdText,
|
|
available,
|
|
modified,
|
|
] of table(zip, entries, "branded_food.csv", [
|
|
"fdc_id",
|
|
"gtin_upc",
|
|
"brand_name",
|
|
"brand_owner",
|
|
"serving_size",
|
|
"serving_size_unit",
|
|
"household_serving_fulltext",
|
|
"available_date",
|
|
"modified_date",
|
|
])) {
|
|
const gtin = databaseGtin(gtinUpc);
|
|
if (!gtin) continue;
|
|
const date = available || modified || "";
|
|
const fdcId = Number(id);
|
|
const current = byGtin.get(gtin);
|
|
if (
|
|
current &&
|
|
(current.date > date ||
|
|
(current.date === date && current.fdcId > fdcId))
|
|
)
|
|
continue;
|
|
const qty = Number(servingSize);
|
|
const household = (householdText || "").trim();
|
|
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
|
|
byGtin.set(keep(gtin), {
|
|
fdcId,
|
|
date: keep(date),
|
|
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
|
|
servingQty: unit && qty > 0 ? qty : null,
|
|
servingUnit: unit || "g",
|
|
serving:
|
|
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
|
|
name: "",
|
|
gtin: "",
|
|
});
|
|
}
|
|
strings.clear();
|
|
const byFdc = new Map();
|
|
for (const [gtin, product] of byGtin) {
|
|
product.gtin = gtin;
|
|
byFdc.set(product.fdcId, product);
|
|
}
|
|
byGtin.clear();
|
|
|
|
for await (const [id, description] of table(zip, entries, "food.csv", [
|
|
"fdc_id",
|
|
"description",
|
|
])) {
|
|
const product = byFdc.get(Number(id));
|
|
if (product)
|
|
product.name = Buffer.from(description.slice(0, 200)).toString();
|
|
}
|
|
|
|
let count = 0;
|
|
const finish = (product, values) => {
|
|
const nutrients = {};
|
|
const late = [];
|
|
for (let i = 0; i < values.length; i += 2) {
|
|
const mapping = mapper.get(values[i]);
|
|
const number = values[i + 1];
|
|
if (!mapping || !Number.isFinite(number) || number < 0) continue;
|
|
if (mapping.fallback) late.push([mapping, number]);
|
|
else {
|
|
const value = mapping.convert(number);
|
|
if (value !== null) nutrients[mapping.key] = value;
|
|
}
|
|
}
|
|
for (const [mapping, number] of late)
|
|
if (nutrients[mapping.key] === undefined)
|
|
nutrients[mapping.key] = mapping.convert(number);
|
|
if (
|
|
nutrients["energy-kcal"] === undefined ||
|
|
nutrients["energy-kcal"] > 950
|
|
)
|
|
return;
|
|
sink.add({
|
|
gtin: product.gtin,
|
|
name: product.name,
|
|
brand: product.brand,
|
|
serving: product.serving,
|
|
servingQty: product.servingQty,
|
|
servingUnit: product.servingUnit,
|
|
nutrients,
|
|
});
|
|
if (++count % 250000 === 0) onProgress?.(count);
|
|
};
|
|
// ~30 million rows, grouped by food: each product is finished as soon as
|
|
// the next one starts, so only one food's nutrients are held at a time.
|
|
let currentId = null;
|
|
let values = [];
|
|
for await (const [id, nutrientId, amount] of table(
|
|
zip,
|
|
entries,
|
|
"food_nutrient.csv",
|
|
["fdc_id", "nutrient_id", "amount"],
|
|
)) {
|
|
const fdcId = Number(id);
|
|
if (fdcId !== currentId) {
|
|
const done = byFdc.get(currentId);
|
|
if (done) {
|
|
finish(done, values);
|
|
byFdc.delete(currentId);
|
|
}
|
|
currentId = fdcId;
|
|
values = [];
|
|
}
|
|
if (byFdc.has(fdcId) && amount !== "")
|
|
values.push(Number(nutrientId), Number(amount));
|
|
}
|
|
const last = byFdc.get(currentId);
|
|
if (last) finish(last, values);
|
|
return count;
|
|
} finally {
|
|
zip.close();
|
|
}
|
|
}
|