import { createGunzip } from "node:zlib"; import { Readable } from "node:stream"; import { databaseGtin } from "../../shared/barcode.js"; import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js"; export const OFF_CSV = "https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz"; export const userAgent = "Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)"; // Columns that are not nutrient amounts, or duplicate one handled separately. const skipped = new Set([ "energy", "energy-kj", "energy-from-fat", "salt", "added-salt", "ph", "alcohol", // % vol in Open Food Facts, not grams "cocoa", "chlorophyl", "collagen-meat-protein-ratio", "glycemic-index", "water-hardness", "acidity", "carbohydrates-total", ]); const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/; /** * Converts Open Food Facts per-100 g values (grams, kcal) into Calio's * stored units. * @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g` */ export function offNutrients(entries) { const raw = new Map(); for (const [key, value] of entries) { const number = typeof value === "number" ? value : Number(value); if (value === "" || value == null || !Number.isFinite(number) || number < 0) continue; raw.set(key, number); } const nutrients = {}; let kcal = raw.get("energy-kcal"); if (kcal === undefined) { const kj = raw.get("energy-kj") ?? raw.get("energy"); if (kj !== undefined) kcal = kj / 4.184; } if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal; let sodium = raw.get("sodium"); if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5; if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000; for (const [key, grams] of raw) { if ( key === "energy-kcal" || key === "sodium" || skipped.has(key) || notNutrient.test(key) || !nutrientKeyPattern.test(key) || grams > 100 // more than 100 g per 100 g is a data-entry error ) continue; const value = fromGrams(key, grams); if (value !== null) nutrients[key] = value; } return nutrients; } function servingUnit(text) { return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "") ? "ml" : "g"; } /** Shapes a CSV row or API product into a products-table row. */ export function offProduct({ code, name, brands, serving, servingQty, nutrients, }) { const gtin = databaseGtin(code); if (!gtin || nutrients["energy-kcal"] === undefined) return null; const qty = Number(servingQty); return { gtin, name: (name || "").trim().slice(0, 200) || null, brand: (brands || "").split(",")[0].trim().slice(0, 100) || null, serving: (serving || "").trim().slice(0, 100) || null, servingQty: Number.isFinite(qty) && qty > 0 ? qty : null, servingUnit: servingUnit(serving), nutrients, }; } async function* lines(stream) { let rest = ""; stream.setEncoding("utf8"); for await (const chunk of stream) { const parts = (rest + chunk).split("\n"); rest = parts.pop(); yield* parts; } if (rest) yield rest; } /** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */ export async function importOffCsv(response, sink, { onProgress } = {}) { const gunzip = Readable.fromWeb(response.body).pipe(createGunzip()); let header; let count = 0; for await (const line of lines(gunzip)) { const fields = line.split("\t"); if (!header) { header = Object.fromEntries(fields.map((name, i) => [name, i])); for (const column of ["code", "product_name", "energy-kcal_100g"]) if (!(column in header)) throw new Error(`Open Food Facts export is missing ${column}`); header.nutrientColumns = fields .map((name, i) => [name, i]) .filter(([name]) => name.endsWith("_100g")) .map(([name, i]) => [name.slice(0, -5), i]); continue; } if (fields[header.no_nutrition_data] === "on") continue; const product = offProduct({ code: fields[header.code], name: fields[header.product_name], brands: fields[header.brands], serving: fields[header.serving_size], servingQty: fields[header.serving_quantity], nutrients: offNutrients( header.nutrientColumns.map(([key, i]) => [key, fields[i]]), ), }); if (!product) continue; sink.add(product); if (++count % 250000 === 0) onProgress?.(count); } if (!header) throw new Error("Open Food Facts export was empty"); return count; } const apiFields = "code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments"; /** Live lookup, used only when no local copy has the product. */ export async function fetchOffProduct(gtin, fetchImpl = fetch) { // Open Food Facts keys products by EAN-13 (or EAN-8). const stripped = gtin.replace(/^0+/, ""); const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0"); const response = await fetchImpl( `https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`, { headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) }, ); if (response.status === 404) return null; if (!response.ok) throw new Error(`Open Food Facts answered ${response.status}`); const data = await response.json(); if (data.status !== 1 || !data.product) return null; const p = data.product; const entries = Object.entries(p.nutriments || {}) .filter(([key]) => key.endsWith("_100g")) .map(([key, value]) => [key.slice(0, -5), value]); const product = offProduct({ code: gtin, name: p.product_name_en || p.product_name, brands: p.brands, serving: p.serving_size, servingQty: p.serving_quantity, nutrients: offNutrients(entries), }); return product && { ...product, source: "api" }; }