Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s

Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-22 13:41:44 -07:00
parent 8f6981c8a2
commit 80fc48b9f1
39 changed files with 4434 additions and 33 deletions

165
server/products/db.js Normal file
View file

@ -0,0 +1,165 @@
import { DatabaseSync } from "node:sqlite";
import { titleCase } from "../../shared/products.js";
// Each source lives in its own table so a refresh can be built alongside the
// old copy and swapped in atomically. The file is a cache: it can be deleted
// at any time and is rebuilt on the next import.
export const sources = ["usda", "off", "api"];
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
export function openProducts(filename) {
const db = new DatabaseSync(filename);
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
for (const source of ["usda", "off"])
db.exec(
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
);
// API answers include misses (nutrients NULL) so repeated scans of an
// unknown code don't hit Open Food Facts again.
db.exec(
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
);
const select = Object.fromEntries(
sources.map((source) => [
source,
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
]),
);
const toProduct = (row, source) =>
row?.nutrients
? {
gtin: row.gtin,
name: source === "usda" ? titleCase(row.name) : row.name,
brand: source === "usda" ? titleCase(row.brand) : row.brand,
serving: row.serving,
servingQty: row.serving_qty,
servingUnit: row.serving_unit,
nutrients: JSON.parse(row.nutrients),
source,
}
: null;
/** The best local match, or `undefined` when no source has an answer. */
function lookup(
gtin,
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
) {
const found = {};
for (const source of sources) found[source] = select[source].get(gtin);
const api = found.api;
if (
api &&
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
)
found.api = undefined;
const usda = toProduct(found.usda, "usda");
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
// USDA comes from manufacturers' own label submissions, so it wins;
// community-entered Open Food Facts fills in what it lacks.
const base = usda || off;
if (base)
return {
...base,
name: base.name || off?.name,
brand: base.brand || off?.brand,
};
return found.api ? null : undefined;
}
const upsertApi = db.prepare(
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
);
function remember(gtin, product) {
upsertApi.run(
gtin,
product?.name ?? null,
product?.brand ?? null,
product?.serving ?? null,
product?.servingQty ?? null,
product?.servingUnit ?? null,
product ? JSON.stringify(product.nutrients) : null,
Date.now(),
);
}
/** Bulk loader used by the importers: writes `<source>_products_next`. */
function beginImport(source) {
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
const insert = db.prepare(
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
);
let pending = 0;
let rows = 0;
db.exec("BEGIN");
return {
add(product) {
insert.run(
product.gtin,
product.name || null,
product.brand || null,
product.serving || null,
product.servingQty ?? null,
product.servingUnit || null,
JSON.stringify(product.nutrients),
);
rows++;
if (++pending >= 20000) {
db.exec("COMMIT; BEGIN");
pending = 0;
}
},
commit(version) {
db.exec(`COMMIT; BEGIN IMMEDIATE;
DROP TABLE IF EXISTS ${source}_products;
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
db.prepare(
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
VALUES (?,?,?,?,?,NULL)`,
).run(
source,
version,
new Date().toISOString(),
rows,
new Date().toISOString(),
);
db.exec("COMMIT");
// Old pages are reused by the next import; reclaiming them would
// rewrite the whole file.
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
return rows;
},
abort() {
if (db.isTransaction) db.exec("ROLLBACK");
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
},
};
}
const state = (source) =>
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
function recordCheck(source, error = null) {
db.prepare(
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
).run(source, new Date().toISOString(), error);
}
const status = () =>
db
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
.all();
return {
db,
lookup,
remember,
beginImport,
state,
recordCheck,
status,
close: () => db.close(),
};
}

94
server/products/import.js Normal file
View file

@ -0,0 +1,94 @@
// Refreshes the local product databases. Runs as a low-priority child process
// (see scheduler.js) so parsing several gigabytes never stalls the app.
// node server/products/import.js <data dir> [--force]
import { mkdirSync, rmSync } from "node:fs";
import { setPriority } from "node:os";
import { join, resolve } from "node:path";
import { openProducts } from "./db.js";
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
const dataDir = resolve(process.argv[2] || "data");
const force = process.argv.includes("--force");
const OFF_MAX_AGE = 7 * 86400000;
const log = (...args) => console.log("[products]", ...args);
try {
setPriority(19);
} catch {
// Lowering priority is best effort.
}
const products = openProducts(join(dataDir, "products.sqlite"));
const tmp = join(dataDir, "tmp");
mkdirSync(tmp, { recursive: true });
async function refreshUsda() {
const latest = await latestUsdaArchive();
if (!force && products.state("usda")?.version === latest.version)
return products.recordCheck("usda");
const file = join(tmp, "usda-branded.zip");
log(`downloading USDA Branded Foods ${latest.version}`);
try {
await download(latest.url, file);
const sink = products.beginImport("usda");
try {
await importUsdaZip(file, sink, {
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
});
log(
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
} finally {
rmSync(file, { force: true });
}
}
async function refreshOff() {
const state = products.state("off");
if (
!force &&
state?.imported_at &&
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
)
return;
const response = await fetch(OFF_CSV, {
headers: { "User-Agent": userAgent },
});
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const version =
response.headers.get("last-modified") || new Date().toUTCString();
if (!force && state?.version === version) {
await response.body?.cancel();
return products.recordCheck("off");
}
log("streaming the Open Food Facts export");
const sink = products.beginImport("off");
try {
await importOffCsv(response, sink, {
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
});
log(
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
}
for (const [source, refresh] of [
["usda", refreshUsda],
["off", refreshOff],
]) {
try {
await refresh();
} catch (error) {
console.error(`[products] ${source} refresh failed:`, error.message);
products.recordCheck(source, String(error.message).slice(0, 300));
}
}
products.close();

175
server/products/off.js Normal file
View file

@ -0,0 +1,175 @@
import { createGunzip } from "node:zlib";
import { Readable } from "node:stream";
import { databaseGtin } from "../../shared/barcode.js";
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
export const OFF_CSV =
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
export const userAgent =
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
// Columns that are not nutrient amounts, or duplicate one handled separately.
const skipped = new Set([
"energy",
"energy-kj",
"energy-from-fat",
"salt",
"added-salt",
"ph",
"alcohol", // % vol in Open Food Facts, not grams
"cocoa",
"chlorophyl",
"collagen-meat-protein-ratio",
"glycemic-index",
"water-hardness",
"acidity",
"carbohydrates-total",
]);
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
/**
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
* stored units.
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
*/
export function offNutrients(entries) {
const raw = new Map();
for (const [key, value] of entries) {
const number = typeof value === "number" ? value : Number(value);
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
continue;
raw.set(key, number);
}
const nutrients = {};
let kcal = raw.get("energy-kcal");
if (kcal === undefined) {
const kj = raw.get("energy-kj") ?? raw.get("energy");
if (kj !== undefined) kcal = kj / 4.184;
}
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
let sodium = raw.get("sodium");
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
for (const [key, grams] of raw) {
if (
key === "energy-kcal" ||
key === "sodium" ||
skipped.has(key) ||
notNutrient.test(key) ||
!nutrientKeyPattern.test(key) ||
grams > 100 // more than 100 g per 100 g is a data-entry error
)
continue;
const value = fromGrams(key, grams);
if (value !== null) nutrients[key] = value;
}
return nutrients;
}
function servingUnit(text) {
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
? "ml"
: "g";
}
/** Shapes a CSV row or API product into a products-table row. */
export function offProduct({
code,
name,
brands,
serving,
servingQty,
nutrients,
}) {
const gtin = databaseGtin(code);
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
const qty = Number(servingQty);
return {
gtin,
name: (name || "").trim().slice(0, 200) || null,
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
serving: (serving || "").trim().slice(0, 100) || null,
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
servingUnit: servingUnit(serving),
nutrients,
};
}
async function* lines(stream) {
let rest = "";
stream.setEncoding("utf8");
for await (const chunk of stream) {
const parts = (rest + chunk).split("\n");
rest = parts.pop();
yield* parts;
}
if (rest) yield rest;
}
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
export async function importOffCsv(response, sink, { onProgress } = {}) {
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
let header;
let count = 0;
for await (const line of lines(gunzip)) {
const fields = line.split("\t");
if (!header) {
header = Object.fromEntries(fields.map((name, i) => [name, i]));
for (const column of ["code", "product_name", "energy-kcal_100g"])
if (!(column in header))
throw new Error(`Open Food Facts export is missing ${column}`);
header.nutrientColumns = fields
.map((name, i) => [name, i])
.filter(([name]) => name.endsWith("_100g"))
.map(([name, i]) => [name.slice(0, -5), i]);
continue;
}
if (fields[header.no_nutrition_data] === "on") continue;
const product = offProduct({
code: fields[header.code],
name: fields[header.product_name],
brands: fields[header.brands],
serving: fields[header.serving_size],
servingQty: fields[header.serving_quantity],
nutrients: offNutrients(
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
),
});
if (!product) continue;
sink.add(product);
if (++count % 250000 === 0) onProgress?.(count);
}
if (!header) throw new Error("Open Food Facts export was empty");
return count;
}
const apiFields =
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
/** Live lookup, used only when no local copy has the product. */
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
// Open Food Facts keys products by EAN-13 (or EAN-8).
const stripped = gtin.replace(/^0+/, "");
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
const response = await fetchImpl(
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
);
if (response.status === 404) return null;
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const data = await response.json();
if (data.status !== 1 || !data.product) return null;
const p = data.product;
const entries = Object.entries(p.nutriments || {})
.filter(([key]) => key.endsWith("_100g"))
.map(([key, value]) => [key.slice(0, -5), value]);
const product = offProduct({
code: gtin,
name: p.product_name_en || p.product_name,
brands: p.brands,
serving: p.serving_size,
servingQty: p.serving_quantity,
nutrients: offNutrients(entries),
});
return product && { ...product, source: "api" };
}

View file

@ -0,0 +1,27 @@
import { fork } from "node:child_process";
import { fileURLToPath } from "node:url";
const script = fileURLToPath(new URL("./import.js", import.meta.url));
const HOUR = 3600000;
/**
* Checks for new product data shortly after start-up and every six hours.
* The import itself decides whether anything needs downloading.
*/
export function scheduleProductImports(dataDir, { firstDelay = 60000 } = {}) {
let child = null;
const run = () => {
if (child) return;
child = fork(script, [dataDir], {
execArgv: ["--max-old-space-size=1024"],
stdio: "inherit",
});
child.on("exit", () => (child = null));
};
const timers = [setTimeout(run, firstDelay), setInterval(run, 6 * HOUR)];
for (const timer of timers) timer.unref();
return () => {
for (const timer of timers) clearInterval(timer);
child?.kill();
};
}

398
server/products/usda.js Normal file
View file

@ -0,0 +1,398 @@
import { createWriteStream } from "node:fs";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import yauzl from "yauzl";
import { databaseGtin } from "../../shared/barcode.js";
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
const archiveName =
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
/** The newest Branded Foods CSV archive linked from the download page. */
export async function latestUsdaArchive(fetchImpl = fetch) {
const response = await fetchImpl(USDA_PAGE, {
signal: AbortSignal.timeout(30000),
});
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
const versions = [...(await response.text()).matchAll(archiveName)].map(
(match) => match[1],
);
if (!versions.length) throw new Error("No USDA branded archive is listed");
const version = versions.sort().at(-1);
return {
version,
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
};
}
export async function download(url, file, fetchImpl = fetch) {
const response = await fetchImpl(url);
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
}
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
// are kept under a slug of their USDA name.
const nutrientIds = {
1008: "energy-kcal",
1003: "proteins",
1004: "fat",
1005: "carbohydrates",
1079: "fiber",
2000: "sugars",
1093: "sodium",
1258: "saturated-fat",
1257: "trans-fat",
1292: "monounsaturated-fat",
1293: "polyunsaturated-fat",
1253: "cholesterol",
1235: "added-sugars",
1086: "polyols",
1082: "soluble-fiber",
1084: "insoluble-fiber",
1009: "starch",
1010: "sucrose",
1011: "glucose",
1012: "fructose",
1013: "lactose",
1014: "maltose",
1075: "galactose",
1056: "sorbitol",
1078: "xylitol",
1055: "mannitol",
1051: "water",
1018: "alcohol",
1057: "caffeine",
1234: "taurine",
1068: "beta-glucan",
1181: "inositol",
1106: "vitamin-a",
1104: "vitamin-a-iu",
1162: "vitamin-c",
1114: "vitamin-d",
1109: "vitamin-e",
1185: "vitamin-k",
1165: "vitamin-b1",
1166: "vitamin-b2",
1167: "vitamin-pp",
1175: "vitamin-b6",
1190: "vitamin-b9",
1177: "folates",
1186: "folic-acid",
1178: "vitamin-b12",
1176: "biotin",
1170: "pantothenic-acid",
1180: "choline",
1087: "calcium",
1089: "iron",
1092: "potassium",
1090: "magnesium",
1091: "phosphorus",
1095: "zinc",
1098: "copper",
1101: "manganese",
1103: "selenium",
1100: "iodine",
1096: "chromium",
1102: "molybdenum",
1088: "chloride",
1099: "fluoride",
};
// Fallbacks used only when the preferred ID is absent.
const fallbacks = {
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
1085: ["fat", (g) => g], // Total fat (NLEA)
1063: ["sugars", (g) => g], // Sugars, Total
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
};
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
const slug = (name) =>
name
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, "")
.slice(0, 48);
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
export function nutrientMapper(nutrientRows) {
const map = new Map();
for (const { id, name, unit } of nutrientRows) {
const from = usdaUnits[unit];
let key = nutrientIds[id];
if (fallbacks[id]) {
const [target, convert] = fallbacks[id];
map.set(id, { key: target, convert, fallback: true });
continue;
}
if (!key) {
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
continue;
key = slug(name);
if (!key) continue;
}
const to = nutrientUnit(key);
const convert =
from === to ? (value) => value : (value) => convertMass(value, from, to);
if (from !== to && convertMass(1, from, to) === null) continue;
map.set(id, { key, convert, fallback: false });
}
return map;
}
// Parses one record starting at `pos`; null when `text` ends mid-record.
function parseRecord(text, pos) {
const fields = [];
for (;;) {
if (pos >= text.length) return null;
if (text[pos] === '"') {
let value = "";
let i = pos + 1;
for (;;) {
const quote = text.indexOf('"', i);
if (quote === -1 || quote + 1 >= text.length) return null;
value += text.slice(i, quote);
if (text[quote + 1] !== '"') {
pos = quote + 1;
break;
}
value += '"'; // "" inside a quoted field
i = quote + 2;
}
fields.push(value);
} else {
let end = pos;
while (end < text.length && text[end] !== "," && text[end] !== "\n")
end++;
if (end >= text.length) return null;
fields.push(text.slice(pos, end).replace(/\r$/, ""));
pos = end;
}
const c = text[pos];
if (c === ",") pos++;
else if (c === "\n") return { fields, next: pos + 1 };
else if (c === "\r" && text[pos + 1] === "\n")
return { fields, next: pos + 2 };
else if (pos + 1 >= text.length) return null;
else pos++; // stray character after a closing quote
}
}
/** Minimal RFC 4180 reader: yields one array of fields per record. */
export async function* csvRecords(stream) {
stream.setEncoding("utf8");
let buffer = "";
for await (const chunk of stream) {
buffer += chunk;
let start = 0;
for (let record; (record = parseRecord(buffer, start)); start = record.next)
yield record.fields;
buffer = buffer.slice(start);
}
if (buffer.trim()) {
const record = parseRecord(buffer + "\n", 0);
if (record) yield record.fields;
}
}
function openZip(file) {
return new Promise((resolve, reject) =>
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
if (error) return reject(error);
const entries = new Map();
zip.on("entry", (entry) =>
entries.set(entry.fileName.split("/").at(-1), entry),
);
zip.on("end", () => resolve({ zip, entries }));
zip.on("error", reject);
}),
);
}
// Yields the requested columns of each row, in the order asked for.
async function* table(zip, entries, name, columns) {
const entry = entries.get(name);
if (!entry) throw new Error(`USDA archive is missing ${name}`);
const stream = await new Promise((resolve, reject) =>
zip.openReadStream(entry, (error, s) =>
error ? reject(error) : resolve(s),
),
);
let indexes;
for await (const fields of csvRecords(stream)) {
if (!indexes) {
indexes = columns.map((column) => fields.indexOf(column));
const missing = columns.filter((_, i) => indexes[i] < 0);
if (missing.length)
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
continue;
}
yield indexes.map((i) => fields[i]);
}
}
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
function servingText(household, qty, unit) {
const amount = `${+qty.toFixed(1)} ${unit}`;
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
return amount;
// Some submissions already include the weight: "3/4 cup (20g)".
return /\d\s*(g|ml|grams?)\b/i.test(household)
? household
: `${household} (${amount})`;
}
/** Reads a Branded Foods CSV archive into `sink`. */
export async function importUsdaZip(file, sink, { onProgress } = {}) {
const { zip, entries } = await openZip(file);
try {
const nutrientRows = [];
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
"id",
"name",
"unit_name",
]))
nutrientRows.push({ id: Number(id), name, unit });
const mapper = nutrientMapper(nutrientRows);
// A GTIN can appear several times as a product is resubmitted; keep the
// newest submission.
const byGtin = new Map();
const strings = new Map();
// Substrings keep their whole parse buffer alive; store fresh copies,
// shared across the many products with the same brand or serving.
const keep = (text) => {
if (!text) return "";
let copy = strings.get(text);
if (copy === undefined) {
copy = Buffer.from(text).toString();
strings.set(copy, copy);
}
return copy;
};
for await (const [
id,
gtinUpc,
brandName,
brandOwner,
servingSize,
servingSizeUnit,
householdText,
available,
modified,
] of table(zip, entries, "branded_food.csv", [
"fdc_id",
"gtin_upc",
"brand_name",
"brand_owner",
"serving_size",
"serving_size_unit",
"household_serving_fulltext",
"available_date",
"modified_date",
])) {
const gtin = databaseGtin(gtinUpc);
if (!gtin) continue;
const date = available || modified || "";
const fdcId = Number(id);
const current = byGtin.get(gtin);
if (
current &&
(current.date > date ||
(current.date === date && current.fdcId > fdcId))
)
continue;
const qty = Number(servingSize);
const household = (householdText || "").trim();
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
byGtin.set(keep(gtin), {
fdcId,
date: keep(date),
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
servingQty: unit && qty > 0 ? qty : null,
servingUnit: unit || "g",
serving:
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
name: "",
gtin: "",
});
}
strings.clear();
const byFdc = new Map();
for (const [gtin, product] of byGtin) {
product.gtin = gtin;
byFdc.set(product.fdcId, product);
}
byGtin.clear();
for await (const [id, description] of table(zip, entries, "food.csv", [
"fdc_id",
"description",
])) {
const product = byFdc.get(Number(id));
if (product)
product.name = Buffer.from(description.slice(0, 200)).toString();
}
let count = 0;
const finish = (product, values) => {
const nutrients = {};
const late = [];
for (let i = 0; i < values.length; i += 2) {
const mapping = mapper.get(values[i]);
const number = values[i + 1];
if (!mapping || !Number.isFinite(number) || number < 0) continue;
if (mapping.fallback) late.push([mapping, number]);
else {
const value = mapping.convert(number);
if (value !== null) nutrients[mapping.key] = value;
}
}
for (const [mapping, number] of late)
if (nutrients[mapping.key] === undefined)
nutrients[mapping.key] = mapping.convert(number);
if (
nutrients["energy-kcal"] === undefined ||
nutrients["energy-kcal"] > 950
)
return;
sink.add({
gtin: product.gtin,
name: product.name,
brand: product.brand,
serving: product.serving,
servingQty: product.servingQty,
servingUnit: product.servingUnit,
nutrients,
});
if (++count % 250000 === 0) onProgress?.(count);
};
// ~30 million rows, grouped by food: each product is finished as soon as
// the next one starts, so only one food's nutrients are held at a time.
let currentId = null;
let values = [];
for await (const [id, nutrientId, amount] of table(
zip,
entries,
"food_nutrient.csv",
["fdc_id", "nutrient_id", "amount"],
)) {
const fdcId = Number(id);
if (fdcId !== currentId) {
const done = byFdc.get(currentId);
if (done) {
finish(done, values);
byFdc.delete(currentId);
}
currentId = fdcId;
values = [];
}
if (byFdc.has(fdcId) && amount !== "")
values.push(Number(nutrientId), Number(amount));
}
const last = byFdc.get(currentId);
if (last) finish(last, values);
return count;
} finally {
zip.close();
}
}