Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f6981c8a2
commit
80fc48b9f1
39 changed files with 4434 additions and 33 deletions
165
server/products/db.js
Normal file
165
server/products/db.js
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { titleCase } from "../../shared/products.js";
|
||||
|
||||
// Each source lives in its own table so a refresh can be built alongside the
|
||||
// old copy and swapped in atomically. The file is a cache: it can be deleted
|
||||
// at any time and is rebuilt on the next import.
|
||||
export const sources = ["usda", "off", "api"];
|
||||
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
|
||||
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
|
||||
|
||||
export function openProducts(filename) {
|
||||
const db = new DatabaseSync(filename);
|
||||
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
|
||||
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
|
||||
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
|
||||
for (const source of ["usda", "off"])
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
|
||||
);
|
||||
// API answers include misses (nutrients NULL) so repeated scans of an
|
||||
// unknown code don't hit Open Food Facts again.
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
|
||||
);
|
||||
const select = Object.fromEntries(
|
||||
sources.map((source) => [
|
||||
source,
|
||||
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
|
||||
]),
|
||||
);
|
||||
const toProduct = (row, source) =>
|
||||
row?.nutrients
|
||||
? {
|
||||
gtin: row.gtin,
|
||||
name: source === "usda" ? titleCase(row.name) : row.name,
|
||||
brand: source === "usda" ? titleCase(row.brand) : row.brand,
|
||||
serving: row.serving,
|
||||
servingQty: row.serving_qty,
|
||||
servingUnit: row.serving_unit,
|
||||
nutrients: JSON.parse(row.nutrients),
|
||||
source,
|
||||
}
|
||||
: null;
|
||||
|
||||
/** The best local match, or `undefined` when no source has an answer. */
|
||||
function lookup(
|
||||
gtin,
|
||||
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
|
||||
) {
|
||||
const found = {};
|
||||
for (const source of sources) found[source] = select[source].get(gtin);
|
||||
const api = found.api;
|
||||
if (
|
||||
api &&
|
||||
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
|
||||
)
|
||||
found.api = undefined;
|
||||
const usda = toProduct(found.usda, "usda");
|
||||
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
|
||||
// USDA comes from manufacturers' own label submissions, so it wins;
|
||||
// community-entered Open Food Facts fills in what it lacks.
|
||||
const base = usda || off;
|
||||
if (base)
|
||||
return {
|
||||
...base,
|
||||
name: base.name || off?.name,
|
||||
brand: base.brand || off?.brand,
|
||||
};
|
||||
return found.api ? null : undefined;
|
||||
}
|
||||
|
||||
const upsertApi = db.prepare(
|
||||
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
|
||||
);
|
||||
function remember(gtin, product) {
|
||||
upsertApi.run(
|
||||
gtin,
|
||||
product?.name ?? null,
|
||||
product?.brand ?? null,
|
||||
product?.serving ?? null,
|
||||
product?.servingQty ?? null,
|
||||
product?.servingUnit ?? null,
|
||||
product ? JSON.stringify(product.nutrients) : null,
|
||||
Date.now(),
|
||||
);
|
||||
}
|
||||
|
||||
/** Bulk loader used by the importers: writes `<source>_products_next`. */
|
||||
function beginImport(source) {
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
|
||||
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
|
||||
const insert = db.prepare(
|
||||
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
|
||||
);
|
||||
let pending = 0;
|
||||
let rows = 0;
|
||||
db.exec("BEGIN");
|
||||
return {
|
||||
add(product) {
|
||||
insert.run(
|
||||
product.gtin,
|
||||
product.name || null,
|
||||
product.brand || null,
|
||||
product.serving || null,
|
||||
product.servingQty ?? null,
|
||||
product.servingUnit || null,
|
||||
JSON.stringify(product.nutrients),
|
||||
);
|
||||
rows++;
|
||||
if (++pending >= 20000) {
|
||||
db.exec("COMMIT; BEGIN");
|
||||
pending = 0;
|
||||
}
|
||||
},
|
||||
commit(version) {
|
||||
db.exec(`COMMIT; BEGIN IMMEDIATE;
|
||||
DROP TABLE IF EXISTS ${source}_products;
|
||||
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
|
||||
db.prepare(
|
||||
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
|
||||
VALUES (?,?,?,?,?,NULL)`,
|
||||
).run(
|
||||
source,
|
||||
version,
|
||||
new Date().toISOString(),
|
||||
rows,
|
||||
new Date().toISOString(),
|
||||
);
|
||||
db.exec("COMMIT");
|
||||
// Old pages are reused by the next import; reclaiming them would
|
||||
// rewrite the whole file.
|
||||
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
||||
return rows;
|
||||
},
|
||||
abort() {
|
||||
if (db.isTransaction) db.exec("ROLLBACK");
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const state = (source) =>
|
||||
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
|
||||
function recordCheck(source, error = null) {
|
||||
db.prepare(
|
||||
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
|
||||
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
|
||||
).run(source, new Date().toISOString(), error);
|
||||
}
|
||||
const status = () =>
|
||||
db
|
||||
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
|
||||
.all();
|
||||
|
||||
return {
|
||||
db,
|
||||
lookup,
|
||||
remember,
|
||||
beginImport,
|
||||
state,
|
||||
recordCheck,
|
||||
status,
|
||||
close: () => db.close(),
|
||||
};
|
||||
}
|
||||
94
server/products/import.js
Normal file
94
server/products/import.js
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
// Refreshes the local product databases. Runs as a low-priority child process
|
||||
// (see scheduler.js) so parsing several gigabytes never stalls the app.
|
||||
// node server/products/import.js <data dir> [--force]
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { setPriority } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { openProducts } from "./db.js";
|
||||
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
|
||||
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
|
||||
|
||||
const dataDir = resolve(process.argv[2] || "data");
|
||||
const force = process.argv.includes("--force");
|
||||
const OFF_MAX_AGE = 7 * 86400000;
|
||||
const log = (...args) => console.log("[products]", ...args);
|
||||
try {
|
||||
setPriority(19);
|
||||
} catch {
|
||||
// Lowering priority is best effort.
|
||||
}
|
||||
const products = openProducts(join(dataDir, "products.sqlite"));
|
||||
const tmp = join(dataDir, "tmp");
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
|
||||
async function refreshUsda() {
|
||||
const latest = await latestUsdaArchive();
|
||||
if (!force && products.state("usda")?.version === latest.version)
|
||||
return products.recordCheck("usda");
|
||||
const file = join(tmp, "usda-branded.zip");
|
||||
log(`downloading USDA Branded Foods ${latest.version}`);
|
||||
try {
|
||||
await download(latest.url, file);
|
||||
const sink = products.beginImport("usda");
|
||||
try {
|
||||
await importUsdaZip(file, sink, {
|
||||
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
rmSync(file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshOff() {
|
||||
const state = products.state("off");
|
||||
if (
|
||||
!force &&
|
||||
state?.imported_at &&
|
||||
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
|
||||
)
|
||||
return;
|
||||
const response = await fetch(OFF_CSV, {
|
||||
headers: { "User-Agent": userAgent },
|
||||
});
|
||||
if (!response.ok)
|
||||
throw new Error(`Open Food Facts answered ${response.status}`);
|
||||
const version =
|
||||
response.headers.get("last-modified") || new Date().toUTCString();
|
||||
if (!force && state?.version === version) {
|
||||
await response.body?.cancel();
|
||||
return products.recordCheck("off");
|
||||
}
|
||||
log("streaming the Open Food Facts export");
|
||||
const sink = products.beginImport("off");
|
||||
try {
|
||||
await importOffCsv(response, sink, {
|
||||
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [source, refresh] of [
|
||||
["usda", refreshUsda],
|
||||
["off", refreshOff],
|
||||
]) {
|
||||
try {
|
||||
await refresh();
|
||||
} catch (error) {
|
||||
console.error(`[products] ${source} refresh failed:`, error.message);
|
||||
products.recordCheck(source, String(error.message).slice(0, 300));
|
||||
}
|
||||
}
|
||||
products.close();
|
||||
175
server/products/off.js
Normal file
175
server/products/off.js
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { createGunzip } from "node:zlib";
|
||||
import { Readable } from "node:stream";
|
||||
import { databaseGtin } from "../../shared/barcode.js";
|
||||
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
|
||||
|
||||
export const OFF_CSV =
|
||||
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
|
||||
export const userAgent =
|
||||
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
|
||||
|
||||
// Columns that are not nutrient amounts, or duplicate one handled separately.
|
||||
const skipped = new Set([
|
||||
"energy",
|
||||
"energy-kj",
|
||||
"energy-from-fat",
|
||||
"salt",
|
||||
"added-salt",
|
||||
"ph",
|
||||
"alcohol", // % vol in Open Food Facts, not grams
|
||||
"cocoa",
|
||||
"chlorophyl",
|
||||
"collagen-meat-protein-ratio",
|
||||
"glycemic-index",
|
||||
"water-hardness",
|
||||
"acidity",
|
||||
"carbohydrates-total",
|
||||
]);
|
||||
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
|
||||
|
||||
/**
|
||||
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
|
||||
* stored units.
|
||||
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
|
||||
*/
|
||||
export function offNutrients(entries) {
|
||||
const raw = new Map();
|
||||
for (const [key, value] of entries) {
|
||||
const number = typeof value === "number" ? value : Number(value);
|
||||
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
|
||||
continue;
|
||||
raw.set(key, number);
|
||||
}
|
||||
const nutrients = {};
|
||||
let kcal = raw.get("energy-kcal");
|
||||
if (kcal === undefined) {
|
||||
const kj = raw.get("energy-kj") ?? raw.get("energy");
|
||||
if (kj !== undefined) kcal = kj / 4.184;
|
||||
}
|
||||
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
|
||||
let sodium = raw.get("sodium");
|
||||
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
|
||||
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
|
||||
for (const [key, grams] of raw) {
|
||||
if (
|
||||
key === "energy-kcal" ||
|
||||
key === "sodium" ||
|
||||
skipped.has(key) ||
|
||||
notNutrient.test(key) ||
|
||||
!nutrientKeyPattern.test(key) ||
|
||||
grams > 100 // more than 100 g per 100 g is a data-entry error
|
||||
)
|
||||
continue;
|
||||
const value = fromGrams(key, grams);
|
||||
if (value !== null) nutrients[key] = value;
|
||||
}
|
||||
return nutrients;
|
||||
}
|
||||
|
||||
function servingUnit(text) {
|
||||
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
|
||||
? "ml"
|
||||
: "g";
|
||||
}
|
||||
|
||||
/** Shapes a CSV row or API product into a products-table row. */
|
||||
export function offProduct({
|
||||
code,
|
||||
name,
|
||||
brands,
|
||||
serving,
|
||||
servingQty,
|
||||
nutrients,
|
||||
}) {
|
||||
const gtin = databaseGtin(code);
|
||||
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
|
||||
const qty = Number(servingQty);
|
||||
return {
|
||||
gtin,
|
||||
name: (name || "").trim().slice(0, 200) || null,
|
||||
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
|
||||
serving: (serving || "").trim().slice(0, 100) || null,
|
||||
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
|
||||
servingUnit: servingUnit(serving),
|
||||
nutrients,
|
||||
};
|
||||
}
|
||||
|
||||
async function* lines(stream) {
|
||||
let rest = "";
|
||||
stream.setEncoding("utf8");
|
||||
for await (const chunk of stream) {
|
||||
const parts = (rest + chunk).split("\n");
|
||||
rest = parts.pop();
|
||||
yield* parts;
|
||||
}
|
||||
if (rest) yield rest;
|
||||
}
|
||||
|
||||
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
|
||||
export async function importOffCsv(response, sink, { onProgress } = {}) {
|
||||
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
|
||||
let header;
|
||||
let count = 0;
|
||||
for await (const line of lines(gunzip)) {
|
||||
const fields = line.split("\t");
|
||||
if (!header) {
|
||||
header = Object.fromEntries(fields.map((name, i) => [name, i]));
|
||||
for (const column of ["code", "product_name", "energy-kcal_100g"])
|
||||
if (!(column in header))
|
||||
throw new Error(`Open Food Facts export is missing ${column}`);
|
||||
header.nutrientColumns = fields
|
||||
.map((name, i) => [name, i])
|
||||
.filter(([name]) => name.endsWith("_100g"))
|
||||
.map(([name, i]) => [name.slice(0, -5), i]);
|
||||
continue;
|
||||
}
|
||||
if (fields[header.no_nutrition_data] === "on") continue;
|
||||
const product = offProduct({
|
||||
code: fields[header.code],
|
||||
name: fields[header.product_name],
|
||||
brands: fields[header.brands],
|
||||
serving: fields[header.serving_size],
|
||||
servingQty: fields[header.serving_quantity],
|
||||
nutrients: offNutrients(
|
||||
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
|
||||
),
|
||||
});
|
||||
if (!product) continue;
|
||||
sink.add(product);
|
||||
if (++count % 250000 === 0) onProgress?.(count);
|
||||
}
|
||||
if (!header) throw new Error("Open Food Facts export was empty");
|
||||
return count;
|
||||
}
|
||||
|
||||
const apiFields =
|
||||
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
|
||||
/** Live lookup, used only when no local copy has the product. */
|
||||
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
|
||||
// Open Food Facts keys products by EAN-13 (or EAN-8).
|
||||
const stripped = gtin.replace(/^0+/, "");
|
||||
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
|
||||
const response = await fetchImpl(
|
||||
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
|
||||
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
|
||||
);
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok)
|
||||
throw new Error(`Open Food Facts answered ${response.status}`);
|
||||
const data = await response.json();
|
||||
if (data.status !== 1 || !data.product) return null;
|
||||
const p = data.product;
|
||||
const entries = Object.entries(p.nutriments || {})
|
||||
.filter(([key]) => key.endsWith("_100g"))
|
||||
.map(([key, value]) => [key.slice(0, -5), value]);
|
||||
const product = offProduct({
|
||||
code: gtin,
|
||||
name: p.product_name_en || p.product_name,
|
||||
brands: p.brands,
|
||||
serving: p.serving_size,
|
||||
servingQty: p.serving_quantity,
|
||||
nutrients: offNutrients(entries),
|
||||
});
|
||||
return product && { ...product, source: "api" };
|
||||
}
|
||||
27
server/products/scheduler.js
Normal file
27
server/products/scheduler.js
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { fork } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("./import.js", import.meta.url));
|
||||
const HOUR = 3600000;
|
||||
|
||||
/**
|
||||
* Checks for new product data shortly after start-up and every six hours.
|
||||
* The import itself decides whether anything needs downloading.
|
||||
*/
|
||||
export function scheduleProductImports(dataDir, { firstDelay = 60000 } = {}) {
|
||||
let child = null;
|
||||
const run = () => {
|
||||
if (child) return;
|
||||
child = fork(script, [dataDir], {
|
||||
execArgv: ["--max-old-space-size=1024"],
|
||||
stdio: "inherit",
|
||||
});
|
||||
child.on("exit", () => (child = null));
|
||||
};
|
||||
const timers = [setTimeout(run, firstDelay), setInterval(run, 6 * HOUR)];
|
||||
for (const timer of timers) timer.unref();
|
||||
return () => {
|
||||
for (const timer of timers) clearInterval(timer);
|
||||
child?.kill();
|
||||
};
|
||||
}
|
||||
398
server/products/usda.js
Normal file
398
server/products/usda.js
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
import { createWriteStream } from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import yauzl from "yauzl";
|
||||
import { databaseGtin } from "../../shared/barcode.js";
|
||||
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
|
||||
|
||||
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
|
||||
const archiveName =
|
||||
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
|
||||
|
||||
/** The newest Branded Foods CSV archive linked from the download page. */
|
||||
export async function latestUsdaArchive(fetchImpl = fetch) {
|
||||
const response = await fetchImpl(USDA_PAGE, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
|
||||
const versions = [...(await response.text()).matchAll(archiveName)].map(
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!versions.length) throw new Error("No USDA branded archive is listed");
|
||||
const version = versions.sort().at(-1);
|
||||
return {
|
||||
version,
|
||||
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function download(url, file, fetchImpl = fetch) {
|
||||
const response = await fetchImpl(url);
|
||||
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
|
||||
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
|
||||
}
|
||||
|
||||
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
|
||||
// are kept under a slug of their USDA name.
|
||||
const nutrientIds = {
|
||||
1008: "energy-kcal",
|
||||
1003: "proteins",
|
||||
1004: "fat",
|
||||
1005: "carbohydrates",
|
||||
1079: "fiber",
|
||||
2000: "sugars",
|
||||
1093: "sodium",
|
||||
1258: "saturated-fat",
|
||||
1257: "trans-fat",
|
||||
1292: "monounsaturated-fat",
|
||||
1293: "polyunsaturated-fat",
|
||||
1253: "cholesterol",
|
||||
1235: "added-sugars",
|
||||
1086: "polyols",
|
||||
1082: "soluble-fiber",
|
||||
1084: "insoluble-fiber",
|
||||
1009: "starch",
|
||||
1010: "sucrose",
|
||||
1011: "glucose",
|
||||
1012: "fructose",
|
||||
1013: "lactose",
|
||||
1014: "maltose",
|
||||
1075: "galactose",
|
||||
1056: "sorbitol",
|
||||
1078: "xylitol",
|
||||
1055: "mannitol",
|
||||
1051: "water",
|
||||
1018: "alcohol",
|
||||
1057: "caffeine",
|
||||
1234: "taurine",
|
||||
1068: "beta-glucan",
|
||||
1181: "inositol",
|
||||
1106: "vitamin-a",
|
||||
1104: "vitamin-a-iu",
|
||||
1162: "vitamin-c",
|
||||
1114: "vitamin-d",
|
||||
1109: "vitamin-e",
|
||||
1185: "vitamin-k",
|
||||
1165: "vitamin-b1",
|
||||
1166: "vitamin-b2",
|
||||
1167: "vitamin-pp",
|
||||
1175: "vitamin-b6",
|
||||
1190: "vitamin-b9",
|
||||
1177: "folates",
|
||||
1186: "folic-acid",
|
||||
1178: "vitamin-b12",
|
||||
1176: "biotin",
|
||||
1170: "pantothenic-acid",
|
||||
1180: "choline",
|
||||
1087: "calcium",
|
||||
1089: "iron",
|
||||
1092: "potassium",
|
||||
1090: "magnesium",
|
||||
1091: "phosphorus",
|
||||
1095: "zinc",
|
||||
1098: "copper",
|
||||
1101: "manganese",
|
||||
1103: "selenium",
|
||||
1100: "iodine",
|
||||
1096: "chromium",
|
||||
1102: "molybdenum",
|
||||
1088: "chloride",
|
||||
1099: "fluoride",
|
||||
};
|
||||
// Fallbacks used only when the preferred ID is absent.
|
||||
const fallbacks = {
|
||||
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
|
||||
1085: ["fat", (g) => g], // Total fat (NLEA)
|
||||
1063: ["sugars", (g) => g], // Sugars, Total
|
||||
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
|
||||
};
|
||||
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
|
||||
const slug = (name) =>
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 48);
|
||||
|
||||
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
|
||||
export function nutrientMapper(nutrientRows) {
|
||||
const map = new Map();
|
||||
for (const { id, name, unit } of nutrientRows) {
|
||||
const from = usdaUnits[unit];
|
||||
let key = nutrientIds[id];
|
||||
if (fallbacks[id]) {
|
||||
const [target, convert] = fallbacks[id];
|
||||
map.set(id, { key: target, convert, fallback: true });
|
||||
continue;
|
||||
}
|
||||
if (!key) {
|
||||
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
|
||||
continue;
|
||||
key = slug(name);
|
||||
if (!key) continue;
|
||||
}
|
||||
const to = nutrientUnit(key);
|
||||
const convert =
|
||||
from === to ? (value) => value : (value) => convertMass(value, from, to);
|
||||
if (from !== to && convertMass(1, from, to) === null) continue;
|
||||
map.set(id, { key, convert, fallback: false });
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Parses one record starting at `pos`; null when `text` ends mid-record.
|
||||
function parseRecord(text, pos) {
|
||||
const fields = [];
|
||||
for (;;) {
|
||||
if (pos >= text.length) return null;
|
||||
if (text[pos] === '"') {
|
||||
let value = "";
|
||||
let i = pos + 1;
|
||||
for (;;) {
|
||||
const quote = text.indexOf('"', i);
|
||||
if (quote === -1 || quote + 1 >= text.length) return null;
|
||||
value += text.slice(i, quote);
|
||||
if (text[quote + 1] !== '"') {
|
||||
pos = quote + 1;
|
||||
break;
|
||||
}
|
||||
value += '"'; // "" inside a quoted field
|
||||
i = quote + 2;
|
||||
}
|
||||
fields.push(value);
|
||||
} else {
|
||||
let end = pos;
|
||||
while (end < text.length && text[end] !== "," && text[end] !== "\n")
|
||||
end++;
|
||||
if (end >= text.length) return null;
|
||||
fields.push(text.slice(pos, end).replace(/\r$/, ""));
|
||||
pos = end;
|
||||
}
|
||||
const c = text[pos];
|
||||
if (c === ",") pos++;
|
||||
else if (c === "\n") return { fields, next: pos + 1 };
|
||||
else if (c === "\r" && text[pos + 1] === "\n")
|
||||
return { fields, next: pos + 2 };
|
||||
else if (pos + 1 >= text.length) return null;
|
||||
else pos++; // stray character after a closing quote
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal RFC 4180 reader: yields one array of fields per record. */
|
||||
export async function* csvRecords(stream) {
|
||||
stream.setEncoding("utf8");
|
||||
let buffer = "";
|
||||
for await (const chunk of stream) {
|
||||
buffer += chunk;
|
||||
let start = 0;
|
||||
for (let record; (record = parseRecord(buffer, start)); start = record.next)
|
||||
yield record.fields;
|
||||
buffer = buffer.slice(start);
|
||||
}
|
||||
if (buffer.trim()) {
|
||||
const record = parseRecord(buffer + "\n", 0);
|
||||
if (record) yield record.fields;
|
||||
}
|
||||
}
|
||||
|
||||
function openZip(file) {
|
||||
return new Promise((resolve, reject) =>
|
||||
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
|
||||
if (error) return reject(error);
|
||||
const entries = new Map();
|
||||
zip.on("entry", (entry) =>
|
||||
entries.set(entry.fileName.split("/").at(-1), entry),
|
||||
);
|
||||
zip.on("end", () => resolve({ zip, entries }));
|
||||
zip.on("error", reject);
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Yields the requested columns of each row, in the order asked for.
|
||||
async function* table(zip, entries, name, columns) {
|
||||
const entry = entries.get(name);
|
||||
if (!entry) throw new Error(`USDA archive is missing ${name}`);
|
||||
const stream = await new Promise((resolve, reject) =>
|
||||
zip.openReadStream(entry, (error, s) =>
|
||||
error ? reject(error) : resolve(s),
|
||||
),
|
||||
);
|
||||
let indexes;
|
||||
for await (const fields of csvRecords(stream)) {
|
||||
if (!indexes) {
|
||||
indexes = columns.map((column) => fields.indexOf(column));
|
||||
const missing = columns.filter((_, i) => indexes[i] < 0);
|
||||
if (missing.length)
|
||||
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
|
||||
continue;
|
||||
}
|
||||
yield indexes.map((i) => fields[i]);
|
||||
}
|
||||
}
|
||||
|
||||
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
|
||||
function servingText(household, qty, unit) {
|
||||
const amount = `${+qty.toFixed(1)} ${unit}`;
|
||||
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
|
||||
return amount;
|
||||
// Some submissions already include the weight: "3/4 cup (20g)".
|
||||
return /\d\s*(g|ml|grams?)\b/i.test(household)
|
||||
? household
|
||||
: `${household} (${amount})`;
|
||||
}
|
||||
|
||||
/** Reads a Branded Foods CSV archive into `sink`. */
|
||||
export async function importUsdaZip(file, sink, { onProgress } = {}) {
|
||||
const { zip, entries } = await openZip(file);
|
||||
try {
|
||||
const nutrientRows = [];
|
||||
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
|
||||
"id",
|
||||
"name",
|
||||
"unit_name",
|
||||
]))
|
||||
nutrientRows.push({ id: Number(id), name, unit });
|
||||
const mapper = nutrientMapper(nutrientRows);
|
||||
|
||||
// A GTIN can appear several times as a product is resubmitted; keep the
|
||||
// newest submission.
|
||||
const byGtin = new Map();
|
||||
const strings = new Map();
|
||||
// Substrings keep their whole parse buffer alive; store fresh copies,
|
||||
// shared across the many products with the same brand or serving.
|
||||
const keep = (text) => {
|
||||
if (!text) return "";
|
||||
let copy = strings.get(text);
|
||||
if (copy === undefined) {
|
||||
copy = Buffer.from(text).toString();
|
||||
strings.set(copy, copy);
|
||||
}
|
||||
return copy;
|
||||
};
|
||||
for await (const [
|
||||
id,
|
||||
gtinUpc,
|
||||
brandName,
|
||||
brandOwner,
|
||||
servingSize,
|
||||
servingSizeUnit,
|
||||
householdText,
|
||||
available,
|
||||
modified,
|
||||
] of table(zip, entries, "branded_food.csv", [
|
||||
"fdc_id",
|
||||
"gtin_upc",
|
||||
"brand_name",
|
||||
"brand_owner",
|
||||
"serving_size",
|
||||
"serving_size_unit",
|
||||
"household_serving_fulltext",
|
||||
"available_date",
|
||||
"modified_date",
|
||||
])) {
|
||||
const gtin = databaseGtin(gtinUpc);
|
||||
if (!gtin) continue;
|
||||
const date = available || modified || "";
|
||||
const fdcId = Number(id);
|
||||
const current = byGtin.get(gtin);
|
||||
if (
|
||||
current &&
|
||||
(current.date > date ||
|
||||
(current.date === date && current.fdcId > fdcId))
|
||||
)
|
||||
continue;
|
||||
const qty = Number(servingSize);
|
||||
const household = (householdText || "").trim();
|
||||
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
|
||||
byGtin.set(keep(gtin), {
|
||||
fdcId,
|
||||
date: keep(date),
|
||||
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
|
||||
servingQty: unit && qty > 0 ? qty : null,
|
||||
servingUnit: unit || "g",
|
||||
serving:
|
||||
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
|
||||
name: "",
|
||||
gtin: "",
|
||||
});
|
||||
}
|
||||
strings.clear();
|
||||
const byFdc = new Map();
|
||||
for (const [gtin, product] of byGtin) {
|
||||
product.gtin = gtin;
|
||||
byFdc.set(product.fdcId, product);
|
||||
}
|
||||
byGtin.clear();
|
||||
|
||||
for await (const [id, description] of table(zip, entries, "food.csv", [
|
||||
"fdc_id",
|
||||
"description",
|
||||
])) {
|
||||
const product = byFdc.get(Number(id));
|
||||
if (product)
|
||||
product.name = Buffer.from(description.slice(0, 200)).toString();
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
const finish = (product, values) => {
|
||||
const nutrients = {};
|
||||
const late = [];
|
||||
for (let i = 0; i < values.length; i += 2) {
|
||||
const mapping = mapper.get(values[i]);
|
||||
const number = values[i + 1];
|
||||
if (!mapping || !Number.isFinite(number) || number < 0) continue;
|
||||
if (mapping.fallback) late.push([mapping, number]);
|
||||
else {
|
||||
const value = mapping.convert(number);
|
||||
if (value !== null) nutrients[mapping.key] = value;
|
||||
}
|
||||
}
|
||||
for (const [mapping, number] of late)
|
||||
if (nutrients[mapping.key] === undefined)
|
||||
nutrients[mapping.key] = mapping.convert(number);
|
||||
if (
|
||||
nutrients["energy-kcal"] === undefined ||
|
||||
nutrients["energy-kcal"] > 950
|
||||
)
|
||||
return;
|
||||
sink.add({
|
||||
gtin: product.gtin,
|
||||
name: product.name,
|
||||
brand: product.brand,
|
||||
serving: product.serving,
|
||||
servingQty: product.servingQty,
|
||||
servingUnit: product.servingUnit,
|
||||
nutrients,
|
||||
});
|
||||
if (++count % 250000 === 0) onProgress?.(count);
|
||||
};
|
||||
// ~30 million rows, grouped by food: each product is finished as soon as
|
||||
// the next one starts, so only one food's nutrients are held at a time.
|
||||
let currentId = null;
|
||||
let values = [];
|
||||
for await (const [id, nutrientId, amount] of table(
|
||||
zip,
|
||||
entries,
|
||||
"food_nutrient.csv",
|
||||
["fdc_id", "nutrient_id", "amount"],
|
||||
)) {
|
||||
const fdcId = Number(id);
|
||||
if (fdcId !== currentId) {
|
||||
const done = byFdc.get(currentId);
|
||||
if (done) {
|
||||
finish(done, values);
|
||||
byFdc.delete(currentId);
|
||||
}
|
||||
currentId = fdcId;
|
||||
values = [];
|
||||
}
|
||||
if (byFdc.has(fdcId) && amount !== "")
|
||||
values.push(Number(nutrientId), Number(amount));
|
||||
}
|
||||
const last = byFdc.get(currentId);
|
||||
if (last) finish(last, values);
|
||||
return count;
|
||||
} finally {
|
||||
zip.close();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue