import { createWriteStream } from "node:fs"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import yauzl from "yauzl"; import { databaseGtin } from "../../shared/barcode.js"; import { convertMass, nutrientUnit } from "../../shared/nutrients.js"; export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets"; const archiveName = /FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g; /** The newest Branded Foods CSV archive linked from the download page. */ export async function latestUsdaArchive(fetchImpl = fetch) { const response = await fetchImpl(USDA_PAGE, { signal: AbortSignal.timeout(30000), }); if (!response.ok) throw new Error(`USDA answered ${response.status}`); const versions = [...(await response.text()).matchAll(archiveName)].map( (match) => match[1], ); if (!versions.length) throw new Error("No USDA branded archive is listed"); const version = versions.sort().at(-1); return { version, url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`, }; } export async function download(url, file, fetchImpl = fetch) { const response = await fetchImpl(url); if (!response.ok) throw new Error(`Download failed: ${response.status}`); await pipeline(Readable.fromWeb(response.body), createWriteStream(file)); } // FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients // are kept under a slug of their USDA name. const nutrientIds = { 1008: "energy-kcal", 1003: "proteins", 1004: "fat", 1005: "carbohydrates", 1079: "fiber", 2000: "sugars", 1093: "sodium", 1258: "saturated-fat", 1257: "trans-fat", 1292: "monounsaturated-fat", 1293: "polyunsaturated-fat", 1253: "cholesterol", 1235: "added-sugars", 1086: "polyols", 1082: "soluble-fiber", 1084: "insoluble-fiber", 1009: "starch", 1010: "sucrose", 1011: "glucose", 1012: "fructose", 1013: "lactose", 1014: "maltose", 1075: "galactose", 1056: "sorbitol", 1078: "xylitol", 1055: "mannitol", 1051: "water", 1018: "alcohol", 1057: "caffeine", 1234: "taurine", 1068: "beta-glucan", 1181: "inositol", 1106: "vitamin-a", 1104: "vitamin-a-iu", 1162: "vitamin-c", 1114: "vitamin-d", 1109: "vitamin-e", 1185: "vitamin-k", 1165: "vitamin-b1", 1166: "vitamin-b2", 1167: "vitamin-pp", 1175: "vitamin-b6", 1190: "vitamin-b9", 1177: "folates", 1186: "folic-acid", 1178: "vitamin-b12", 1176: "biotin", 1170: "pantothenic-acid", 1180: "choline", 1087: "calcium", 1089: "iron", 1092: "potassium", 1090: "magnesium", 1091: "phosphorus", 1095: "zinc", 1098: "copper", 1101: "manganese", 1103: "selenium", 1100: "iodine", 1096: "chromium", 1102: "molybdenum", 1088: "chloride", 1099: "fluoride", }; // Fallbacks used only when the preferred ID is absent. const fallbacks = { 1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ 1085: ["fat", (g) => g], // Total fat (NLEA) 1063: ["sugars", (g) => g], // Sugars, Total 1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU }; const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" }; const slug = (name) => name .toLowerCase() .replace(/[^a-z0-9]+/g, "-") .replace(/^-|-$/g, "") .slice(0, 48); /** Builds a converter from a nutrient row (id, amount) to [key, value]. */ export function nutrientMapper(nutrientRows) { const map = new Map(); for (const { id, name, unit } of nutrientRows) { const from = usdaUnits[unit]; let key = nutrientIds[id]; if (fallbacks[id]) { const [target, convert] = fallbacks[id]; map.set(id, { key: target, convert, fallback: true }); continue; } if (!key) { if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name)) continue; key = slug(name); if (!key) continue; } const to = nutrientUnit(key); const convert = from === to ? (value) => value : (value) => convertMass(value, from, to); if (from !== to && convertMass(1, from, to) === null) continue; map.set(id, { key, convert, fallback: false }); } return map; } // Parses one record starting at `pos`; null when `text` ends mid-record. function parseRecord(text, pos) { const fields = []; for (;;) { if (pos >= text.length) return null; if (text[pos] === '"') { let value = ""; let i = pos + 1; for (;;) { const quote = text.indexOf('"', i); if (quote === -1 || quote + 1 >= text.length) return null; value += text.slice(i, quote); if (text[quote + 1] !== '"') { pos = quote + 1; break; } value += '"'; // "" inside a quoted field i = quote + 2; } fields.push(value); } else { let end = pos; while (end < text.length && text[end] !== "," && text[end] !== "\n") end++; if (end >= text.length) return null; fields.push(text.slice(pos, end).replace(/\r$/, "")); pos = end; } const c = text[pos]; if (c === ",") pos++; else if (c === "\n") return { fields, next: pos + 1 }; else if (c === "\r" && text[pos + 1] === "\n") return { fields, next: pos + 2 }; else if (pos + 1 >= text.length) return null; else pos++; // stray character after a closing quote } } /** Minimal RFC 4180 reader: yields one array of fields per record. */ export async function* csvRecords(stream) { stream.setEncoding("utf8"); let buffer = ""; for await (const chunk of stream) { buffer += chunk; let start = 0; for (let record; (record = parseRecord(buffer, start)); start = record.next) yield record.fields; buffer = buffer.slice(start); } if (buffer.trim()) { const record = parseRecord(buffer + "\n", 0); if (record) yield record.fields; } } function openZip(file) { return new Promise((resolve, reject) => yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => { if (error) return reject(error); const entries = new Map(); zip.on("entry", (entry) => entries.set(entry.fileName.split("/").at(-1), entry), ); zip.on("end", () => resolve({ zip, entries })); zip.on("error", reject); }), ); } // Yields the requested columns of each row, in the order asked for. async function* table(zip, entries, name, columns) { const entry = entries.get(name); if (!entry) throw new Error(`USDA archive is missing ${name}`); const stream = await new Promise((resolve, reject) => zip.openReadStream(entry, (error, s) => error ? reject(error) : resolve(s), ), ); let indexes; for await (const fields of csvRecords(stream)) { if (!indexes) { indexes = columns.map((column) => fields.indexOf(column)); const missing = columns.filter((_, i) => indexes[i] < 0); if (missing.length) throw new Error(`USDA ${name} is missing ${missing.join(", ")}`); continue; } yield indexes.map((i) => fields[i]); } } const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" }; function servingText(household, qty, unit) { const amount = `${+qty.toFixed(1)} ${unit}`; if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household)) return amount; // Some submissions already include the weight: "3/4 cup (20g)". return /\d\s*(g|ml|grams?)\b/i.test(household) ? household : `${household} (${amount})`; } /** Reads a Branded Foods CSV archive into `sink`. */ export async function importUsdaZip(file, sink, { onProgress } = {}) { const { zip, entries } = await openZip(file); try { const nutrientRows = []; for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [ "id", "name", "unit_name", ])) nutrientRows.push({ id: Number(id), name, unit }); const mapper = nutrientMapper(nutrientRows); // A GTIN can appear several times as a product is resubmitted; keep the // newest submission. const byGtin = new Map(); const strings = new Map(); // Substrings keep their whole parse buffer alive; store fresh copies, // shared across the many products with the same brand or serving. const keep = (text) => { if (!text) return ""; let copy = strings.get(text); if (copy === undefined) { copy = Buffer.from(text).toString(); strings.set(copy, copy); } return copy; }; for await (const [ id, gtinUpc, brandName, brandOwner, servingSize, servingSizeUnit, householdText, available, modified, ] of table(zip, entries, "branded_food.csv", [ "fdc_id", "gtin_upc", "brand_name", "brand_owner", "serving_size", "serving_size_unit", "household_serving_fulltext", "available_date", "modified_date", ])) { const gtin = databaseGtin(gtinUpc); if (!gtin) continue; const date = available || modified || ""; const fdcId = Number(id); const current = byGtin.get(gtin); if ( current && (current.date > date || (current.date === date && current.fdcId > fdcId)) ) continue; const qty = Number(servingSize); const household = (householdText || "").trim(); const unit = servingUnits[(servingSizeUnit || "").toLowerCase()]; byGtin.set(keep(gtin), { fdcId, date: keep(date), brand: keep((brandName || brandOwner || "").trim().slice(0, 100)), servingQty: unit && qty > 0 ? qty : null, servingUnit: unit || "g", serving: unit && qty > 0 ? keep(servingText(household, qty, unit)) : null, name: "", gtin: "", }); } strings.clear(); const byFdc = new Map(); for (const [gtin, product] of byGtin) { product.gtin = gtin; byFdc.set(product.fdcId, product); } byGtin.clear(); for await (const [id, description] of table(zip, entries, "food.csv", [ "fdc_id", "description", ])) { const product = byFdc.get(Number(id)); if (product) product.name = Buffer.from(description.slice(0, 200)).toString(); } let count = 0; const finish = (product, values) => { const nutrients = {}; const late = []; for (let i = 0; i < values.length; i += 2) { const mapping = mapper.get(values[i]); const number = values[i + 1]; if (!mapping || !Number.isFinite(number) || number < 0) continue; if (mapping.fallback) late.push([mapping, number]); else { const value = mapping.convert(number); if (value !== null) nutrients[mapping.key] = value; } } for (const [mapping, number] of late) if (nutrients[mapping.key] === undefined) nutrients[mapping.key] = mapping.convert(number); if ( nutrients["energy-kcal"] === undefined || nutrients["energy-kcal"] > 950 ) return; sink.add({ gtin: product.gtin, name: product.name, brand: product.brand, serving: product.serving, servingQty: product.servingQty, servingUnit: product.servingUnit, nutrients, }); if (++count % 250000 === 0) onProgress?.(count); }; // ~30 million rows, grouped by food: each product is finished as soon as // the next one starts, so only one food's nutrients are held at a time. let currentId = null; let values = []; for await (const [id, nutrientId, amount] of table( zip, entries, "food_nutrient.csv", ["fdc_id", "nutrient_id", "amount"], )) { const fdcId = Number(id); if (fdcId !== currentId) { const done = byFdc.get(currentId); if (done) { finish(done, values); byFdc.delete(currentId); } currentId = fdcId; values = []; } if (byFdc.has(fdcId) && amount !== "") values.push(Number(nutrientId), Number(amount)); } const last = byFdc.get(currentId); if (last) finish(last, values); return count; } finally { zip.close(); } }