Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s

Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-22 13:41:44 -07:00
parent 8f6981c8a2
commit 80fc48b9f1
39 changed files with 4434 additions and 33 deletions

View file

@ -6,7 +6,11 @@ import { resolve, join } from "node:path";
import { existsSync } from "node:fs";
import { z } from "zod";
import { recordSchema } from "../shared/domain.js";
import { normalizeBarcode } from "../shared/barcode.js";
import { productToFood } from "../shared/products.js";
import { checkPassword, tokenHash } from "./auth.js";
import { fetchOffProduct } from "./products/off.js";
import { parseLabel } from "./ocr/label.js";
export function createApp({
store,
@ -15,6 +19,9 @@ export function createApp({
origin,
secureCookies = true,
dist = resolve("dist"),
products,
fetchProduct = fetchOffProduct,
ocr,
}) {
const app = express();
app.disable("x-powered-by");
@ -22,9 +29,10 @@ export function createApp({
helmet({
contentSecurityPolicy: {
directives: {
"script-src": ["'self'"],
// The barcode reader is WebAssembly, served from this origin.
"script-src": ["'self'", "'wasm-unsafe-eval'"],
"style-src": ["'self'", "'unsafe-inline'"],
"img-src": ["'self'", "data:"],
"img-src": ["'self'", "data:", "blob:"],
"connect-src": ["'self'"],
"object-src": ["'none'"],
"upgrade-insecure-requests": secureCookies ? [] : null,
@ -146,6 +154,77 @@ export function createApp({
});
}
});
app.get("/api/products/:code", requireAuth, async (req, res) => {
const gtin = normalizeBarcode(req.params.code);
if (!gtin)
return res
.status(400)
.json({ error: "That isn’t a valid product barcode." });
if (!products)
return res
.status(503)
.json({ error: "The product database isn’t available." });
let product = products.lookup(gtin);
if (product === undefined) {
try {
product = await fetchProduct(gtin);
} catch {
return res.status(502).json({
error:
"This product isn’t in the local database yet, and Open Food Facts couldn’t be reached.",
gtin,
});
}
products.remember(gtin, product);
}
if (!product)
return res
.status(404)
.json({ error: "No nutrition information for this barcode.", gtin });
res.json({ food: productToFood(product) });
});
app.get("/api/products", requireAuth, (_, res) =>
res.json({ sources: products?.status() || [] }),
);
// One label at a time: recognition already uses every core.
let labelQueue = Promise.resolve();
app.post(
"/api/label",
requireAuth,
express.raw({
type: ["image/jpeg", "image/png", "image/webp"],
limit: "16mb",
}),
async (req, res) => {
if (!Buffer.isBuffer(req.body) || !req.body.length)
return res
.status(415)
.json({ error: "Send a JPEG, PNG or WebP photo of the label." });
if (!ocr)
return res
.status(503)
.json({ error: "Label scanning isn’t available on this server." });
const run = labelQueue.then(async () => parseLabel(await ocr(req.body)));
labelQueue = run.catch(() => {});
try {
const label = await run;
if (!label.found.length)
return res.status(422).json({
error:
"No nutrition facts were found. Fill the frame with the label and keep it sharp.",
});
res.json({ label });
} catch (error) {
if (error?.unavailable)
return res
.status(503)
.json({ error: "Label scanning isn’t available on this server." });
res
.status(422)
.json({ error: "That photo couldn’t be read. Try another one." });
}
},
);
app.use("/api", (_, res) => res.status(404).json({ error: "Not found" }));
if (existsSync(dist)) {
// The service worker is also served without a file extension. Reverse

View file

@ -3,6 +3,9 @@ import { resolve } from "node:path";
import { openDatabase } from "./db.js";
import { createApp } from "./app.js";
import { hashPassword } from "./auth.js";
import { openProducts } from "./products/db.js";
import { scheduleProductImports } from "./products/scheduler.js";
import { createOcr } from "./ocr/engine.js";
const secret = (name) =>
process.env[name + "_FILE"]
? readFileSync(process.env[name + "_FILE"], "utf8").trimEnd()
@ -30,12 +33,30 @@ if (
const dataDir = resolve(process.env.DATA_DIR || "data");
mkdirSync(dataDir, { recursive: true });
const store = openDatabase(resolve(dataDir, "calio.sqlite"));
const products = openProducts(resolve(dataDir, "products.sqlite"));
const stopImports = scheduleProductImports(dataDir);
// Models load in the background so the first label scan doesn't wait.
const engine = createOcr(resolve(process.env.MODELS_DIR || "models")).catch(
(error) => {
console.error("Label scanning is unavailable:", error.message);
return null;
},
);
const app = createApp({
store,
username: process.env.CALIO_USERNAME || "calio",
passwordHash,
origin,
secureCookies,
products,
ocr: async (image) => {
const ocr = await engine;
if (!ocr)
throw Object.assign(new Error("OCR models are not loaded"), {
unavailable: true,
});
return ocr.read(image);
},
});
const server = app.listen(Number(process.env.PORT || 3000), "0.0.0.0", () =>
console.log(`Calio listening at ${origin}`),
@ -43,6 +64,8 @@ const server = app.listen(Number(process.env.PORT || 3000), "0.0.0.0", () =>
for (const signal of ["SIGTERM", "SIGINT"])
process.on(signal, () =>
server.close(() => {
stopImports();
products.close();
store.close();
process.exit(0);
}),

434
server/ocr/engine.js Normal file
View file

@ -0,0 +1,434 @@
// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors
// PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with
// unclip, then CTC decoding of each line crop.
import { readFileSync } from "node:fs";
import { join } from "node:path";
import ort from "onnxruntime-node";
import sharp from "sharp";
const DET_THRESH = 0.3;
const BOX_THRESH = 0.6;
const UNCLIP_RATIO = 1.5;
const REC_HEIGHT = 48;
const REC_BATCH = 8;
/** Reads the CTC character list out of a PaddleOCR inference.yml. */
export function readDictionary(yaml) {
const lines = yaml.split(/\r?\n/);
const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line));
if (start < 0) throw new Error("No character_dict in model config");
const chars = [];
for (const line of lines.slice(start + 1)) {
const match = /^\s*- (.*)$/.exec(line);
if (!match) break;
let value = match[1];
if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'");
else if (value.startsWith('"')) value = JSON.parse(value);
chars.push(value);
}
return ["", ...chars, " "]; // blank, dictionary, then the space character
}
// ---------- geometry ----------
function convexHull(points) {
points.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
const cross = (o, a, b) =>
(a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]);
const lower = [];
for (const p of points) {
while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0)
lower.pop();
lower.push(p);
}
const upper = [];
for (let i = points.length - 1; i >= 0; i--) {
const p = points[i];
while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0)
upper.pop();
upper.push(p);
}
return lower.slice(0, -1).concat(upper.slice(0, -1));
}
/** Minimum-area rectangle as centre, unit axes and side lengths. */
export function minAreaRect(points) {
const hull = convexHull(points);
if (hull.length < 3) {
const [x, y] = hull[0] || [0, 0];
return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 };
}
let best = null;
for (let i = 0; i < hull.length; i++) {
const [x1, y1] = hull[i];
const [x2, y2] = hull[(i + 1) % hull.length];
const length = Math.hypot(x2 - x1, y2 - y1);
if (!length) continue;
const ux = (x2 - x1) / length;
const uy = (y2 - y1) / length;
let minU = Infinity;
let maxU = -Infinity;
let minV = Infinity;
let maxV = -Infinity;
for (const [x, y] of hull) {
const u = x * ux + y * uy;
const v = -x * uy + y * ux;
minU = Math.min(minU, u);
maxU = Math.max(maxU, u);
minV = Math.min(minV, v);
maxV = Math.max(maxV, v);
}
const area = (maxU - minU) * (maxV - minV);
if (!best || area < best.area) {
const cu = (minU + maxU) / 2;
const cv = (minV + maxV) / 2;
best = {
area,
cx: cu * ux - cv * uy,
cy: cu * uy + cv * ux,
ux,
uy,
w: maxU - minU,
h: maxV - minV,
};
}
}
return best;
}
/**
* Orients a rectangle for reading: the axis nearer horizontal runs left to
* right, the other top to bottom. Returns the top-left corner and axes.
*/
function readingFrame({ cx, cy, ux, uy, w, h }) {
let ax = [ux, uy, w];
let bx = [-uy, ux, h];
if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax];
if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]];
if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]];
const [dx, dy, width] = ax;
const [ex, ey, height] = bx;
return {
x: cx - (dx * width) / 2 - (ex * height) / 2,
y: cy - (dy * width) / 2 - (ey * height) / 2,
dx,
dy,
ex,
ey,
width,
height,
};
}
// ---------- detection ----------
function detectionBoxes(prob, width, height) {
const visited = new Uint8Array(width * height);
const boxes = [];
const stack = [];
for (let start = 0; start < prob.length; start++) {
if (visited[start] || prob[start] <= DET_THRESH) continue;
// Flood-fill one connected region, keeping each row's extreme pixels as
// hull candidates.
const rows = new Map();
let sum = 0;
let count = 0;
stack.push(start);
visited[start] = 1;
while (stack.length) {
const index = stack.pop();
const x = index % width;
const y = (index - x) / width;
sum += prob[index];
count++;
const row = rows.get(y);
if (!row) rows.set(y, [x, x]);
else {
if (x < row[0]) row[0] = x;
if (x > row[1]) row[1] = x;
}
for (const next of [
x > 0 ? index - 1 : -1,
x < width - 1 ? index + 1 : -1,
y > 0 ? index - width : -1,
y < height - 1 ? index + width : -1,
])
if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) {
visited[next] = 1;
stack.push(next);
}
}
if (count < 4 || sum / count < BOX_THRESH) continue;
const points = [];
for (const [y, [x0, x1]] of rows)
points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]);
const rect = minAreaRect(points);
if (Math.min(rect.w, rect.h) < 3) continue;
// Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper.
const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h));
rect.w += 2 * d;
rect.h += 2 * d;
if (Math.min(rect.w, rect.h) < 5) continue;
boxes.push({ ...rect, score: sum / count });
}
return boxes;
}
// ---------- recognition ----------
/** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */
function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) {
const { data, width, height } = image;
const plane = REC_HEIGHT * batchWidth;
const scale = frame.height / REC_HEIGHT;
const taps = Math.min(4, Math.max(1, Math.round(scale)));
const pixel = (x, y, c) => {
const xi = Math.min(width - 1, Math.max(0, x));
const yi = Math.min(height - 1, Math.max(0, y));
return data[(yi * width + xi) * 3 + c];
};
for (let ty = 0; ty < REC_HEIGHT; ty++)
for (let tx = 0; tx < cropWidth; tx++) {
let r = 0;
let g = 0;
let b = 0;
for (let sy = 0; sy < taps; sy++)
for (let sx = 0; sx < taps; sx++) {
const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width;
const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height;
const x = frame.x + frame.dx * u + frame.ex * v - 0.5;
const y = frame.y + frame.dy * u + frame.ey * v - 0.5;
const x0 = Math.floor(x);
const y0 = Math.floor(y);
const fx = x - x0;
const fy = y - y0;
for (let c = 0; c < 3; c++) {
const value =
pixel(x0, y0, c) * (1 - fx) * (1 - fy) +
pixel(x0 + 1, y0, c) * fx * (1 - fy) +
pixel(x0, y0 + 1, c) * (1 - fx) * fy +
pixel(x0 + 1, y0 + 1, c) * fx * fy;
if (c === 0) r += value;
else if (c === 1) g += value;
else b += value;
}
}
const n = taps * taps * 127.5;
const at = offset + ty * batchWidth + tx;
out[at] = b / n - 1;
out[at + plane] = g / n - 1;
out[at + 2 * plane] = r / n - 1;
}
}
function ctcDecode(output, dims, chars) {
const [count, steps, classes] = dims;
const results = [];
for (let n = 0; n < count; n++) {
let text = "";
let confidence = 0;
let kept = 0;
let previous = -1;
for (let t = 0; t < steps; t++) {
const base = (n * steps + t) * classes;
let best = 0;
let bestValue = output[base];
for (let c = 1; c < classes; c++)
if (output[base + c] > bestValue) {
best = c;
bestValue = output[base + c];
}
if (best !== 0 && best !== previous) {
text += chars[best] ?? "";
confidence += bestValue;
kept++;
}
previous = best;
}
results.push({ text, score: kept ? confidence / kept : 0 });
}
return results;
}
// ---------- pipeline ----------
export async function createOcr(modelDir, { detSide = 1600 } = {}) {
const options = {
graphOptimizationLevel: "all",
// The arena would keep ~2 GB of detection buffers after the first scan.
enableCpuMemArena: false,
};
const [det, rec] = await Promise.all([
ort.InferenceSession.create(
join(modelDir, "det", "inference.onnx"),
options,
),
ort.InferenceSession.create(
join(modelDir, "rec", "inference.onnx"),
options,
),
]);
const chars = readDictionary(
readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"),
);
async function detect(image) {
// Small photos are enlarged too: tiny print detects far better.
const ratio = Math.min(2, detSide / Math.max(image.width, image.height));
const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32);
const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
const { data } = await sharp(image.data, {
raw: { width: image.width, height: image.height, channels: 3 },
})
.resize(w, h, { fit: "fill" })
.raw()
.toBuffer({ resolveWithObject: true });
const input = new Float32Array(3 * w * h);
const mean = [0.406, 0.456, 0.485]; // applied to B, G, R
const std = [0.225, 0.224, 0.229];
for (let i = 0; i < w * h; i++)
for (let c = 0; c < 3; c++)
// Channel c of the tensor is BGR; data is RGB.
input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c];
const result = await det.run({
[det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]),
});
const prob = result[det.outputNames[0]].data;
const sx = image.width / w;
const sy = image.height / h;
return detectionBoxes(prob, w, h).map((box) => {
// Scale the rectangle back to the full image.
const ax = box.ux * sx;
const ay = box.uy * sy;
const length = Math.hypot(ax, ay);
const bx = -box.uy * sx;
const by = box.ux * sy;
return {
cx: box.cx * sx,
cy: box.cy * sy,
ux: ax / length,
uy: ay / length,
w: box.w * length,
h: box.h * Math.hypot(bx, by),
score: box.score,
};
});
}
async function recognize(image, boxes) {
const frames = boxes.map((box) => {
const frame = readingFrame(box);
return { frame, ratio: frame.width / Math.max(1, frame.height) };
});
const order = frames
.map((_, i) => i)
.sort((a, b) => frames[a].ratio - frames[b].ratio);
const texts = new Array(frames.length);
for (let start = 0; start < order.length; start += REC_BATCH) {
const batch = order.slice(start, start + REC_BATCH);
const maxRatio = Math.max(
320 / REC_HEIGHT,
...batch.map((i) => frames[i].ratio),
);
const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio));
const plane = REC_HEIGHT * batchWidth;
const input = new Float32Array(batch.length * 3 * plane);
batch.forEach((i, n) => {
const cropWidth = Math.min(
batchWidth,
Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)),
);
sampleCrop(
image,
frames[i].frame,
input,
n * 3 * plane,
batchWidth,
cropWidth,
);
});
const result = await rec.run({
[rec.inputNames[0]]: new ort.Tensor("float32", input, [
batch.length,
3,
REC_HEIGHT,
batchWidth,
]),
});
const output = result[rec.outputNames[0]];
ctcDecode(output.data, output.dims, chars).forEach((value, n) => {
texts[batch[n]] = value;
});
}
return boxes.map((box, i) => {
const { frame } = frames[i];
const xs = [
frame.x,
frame.x + frame.dx * frame.width,
frame.x + frame.ex * frame.height,
frame.x + frame.dx * frame.width + frame.ex * frame.height,
];
const ys = [
frame.y,
frame.y + frame.dy * frame.width,
frame.y + frame.ey * frame.height,
frame.y + frame.dy * frame.width + frame.ey * frame.height,
];
return {
text: texts[i].text,
score: texts[i].score,
angle: Math.atan2(frame.dy, frame.dx),
cx: (Math.min(...xs) + Math.max(...xs)) / 2,
cy: (Math.min(...ys) + Math.max(...ys)) / 2,
x0: Math.min(...xs),
y0: Math.min(...ys),
x1: Math.max(...xs),
y1: Math.max(...ys),
height: frame.height,
};
});
}
async function decode(buffer, rotate = 0) {
const { data, info } = await sharp(buffer, { failOn: "none" })
.rotate() // honour EXIF orientation
.rotate(rotate)
.resize(2400, 2400, { fit: "inside", withoutEnlargement: true })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
return { data, width: info.width, height: info.height };
}
async function readImage(image) {
const boxes = await detect(image);
const lines = (await recognize(image, boxes)).filter((line) =>
line.text.trim(),
);
const score = lines.reduce(
(sum, line) => sum + line.score * line.text.length,
0,
);
return { lines, score, boxes };
}
/** Returns recognised lines with their bounding boxes, in image pixels. */
async function read(buffer) {
const image = await decode(buffer);
let best = await readImage(image);
// A label photographed sideways yields mostly tall, narrow boxes; try
// both quarter turns and keep whichever reads more confidently.
const tall = best.boxes.filter((box) => {
const frame = readingFrame(box);
return frame.height > frame.width * 1.5;
}).length;
if (best.boxes.length && tall > best.boxes.length / 2)
for (const angle of [90, 270]) {
const attempt = await readImage(await decode(buffer, angle));
if (attempt.score > best.score) best = attempt;
}
return best.lines;
}
return { read };
}

461
server/ocr/label.js Normal file
View file

@ -0,0 +1,461 @@
// Turns OCR text boxes from a Nutrition Facts panel into a food. Built for
// US/Canadian labels, with the common EU "per 100 g" table as a fallback.
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
// FDA Daily Values (2016 rule) in the unit printed on labels; a %DV next to
// an amount is used to catch misread digits and to fill amounts that are only
// given as a percentage.
const dailyValues = {
fat: [78, "g"],
"saturated-fat": [20, "g"],
cholesterol: [300, "mg"],
sodium: [2300, "mg"],
carbohydrates: [275, "g"],
fiber: [28, "g"],
"added-sugars": [50, "g"],
proteins: [50, "g"],
"vitamin-d": [20, "mcg"],
calcium: [1300, "mg"],
iron: [18, "mg"],
potassium: [4700, "mg"],
"vitamin-a": [900, "mcg"],
"vitamin-c": [90, "mg"],
"vitamin-e": [15, "mg"],
"vitamin-k": [120, "mcg"],
"vitamin-b1": [1.2, "mg"],
"vitamin-b2": [1.3, "mg"],
"vitamin-pp": [16, "mg"],
"vitamin-b6": [1.7, "mg"],
"vitamin-b9": [400, "mcg"],
"vitamin-b12": [2.4, "mcg"],
biotin: [30, "mcg"],
"pantothenic-acid": [5, "mg"],
phosphorus: [1250, "mg"],
iodine: [150, "mcg"],
magnesium: [420, "mg"],
zinc: [11, "mg"],
selenium: [55, "mcg"],
copper: [0.9, "mg"],
manganese: [2.3, "mg"],
chromium: [35, "mcg"],
molybdenum: [45, "mcg"],
chloride: [2300, "mg"],
choline: [550, "mg"],
};
// Most specific first: "saturated fat" must claim its row before "fat" does.
const fields = [
["trans-fat", /\btrans\.?\s*fat|\bgrasas?\s*trans\b/],
[
"saturated-fat",
/\b(?:of which\s*)?sat(?:urated|\.)?\s*fat|\bsaturates\b|\bgrasas?\s*saturadas?\b/,
],
["polyunsaturated-fat", /\bpoly(?:unsat(?:urated|\.)?)?\s*fat/],
["monounsaturated-fat", /\bmono(?:unsat(?:urated|\.)?)?\s*fat/],
[
"fat",
/\b(?:total\s*)?fat\b(?!\s*free)|\bgrasas?(?:\s*totales?)?(?:\s*\(?lipidos\)?)?/,
],
["cholesterol", /\bcholest(?:erol|\.)?|\bcolesterol\b/],
["sodium", /\bsodium\b|\bsodio\b/],
["salt", /\bsalt\b/],
["soluble-fiber", /(?<!in)\bsoluble\s*fib(?:er|re)/],
["insoluble-fiber", /\binsoluble\s*fib(?:er|re)/],
["fiber", /\b(?:dietary\s*)?fib(?:er|re)\b|\bfibra(?:\s*dietetica)?\b/],
["polyols", /\bsugar\s*alcohols?\b|\bpolyols\b/],
[
"sugars",
/\b(?:total\s*|of which\s*)?sugars?\b|\bazucares(?:\s*totales)?\b(?!\s*anadidos)/,
],
[
"carbohydrates",
/\b(?:total\s*)?carb(?:ohydrates?|s|\.)?|\bcarbohidratos(?:\s*\(?hidratos\s*(?:\d\s*)?de\s*carbono\)?)?|\bhidratos\s*de\s*carbono\b/,
],
["proteins", /\bprotein\b|\bproteinas?\b/],
["vitamin-d", /\bvit(?:amin|amina|\.)?\s*d\d?\b/],
["vitamin-a", /\bvit(?:amin|\.)?\s*a\b/],
["vitamin-c", /\bvit(?:amin|\.)?\s*c\b/],
["vitamin-e", /\bvit(?:amin|\.)?\s*e\b/],
["vitamin-k", /\bvit(?:amin|\.)?\s*k\d?\b/],
["vitamin-b6", /\bvit(?:amin|\.)?\s*b\s*-?6\b/],
["vitamin-b12", /\bvit(?:amin|\.)?\s*b\s*-?12\b/],
["vitamin-b1", /\bthiamine?\b/],
["vitamin-b2", /\briboflavin\b/],
["vitamin-pp", /\bniacin\b/],
["vitamin-b9", /\bfolate\b|\bfolic acid\b/],
["biotin", /\bbiotin\b/],
["pantothenic-acid", /\bpantothenic\b/],
["calcium", /\bcalcium\b|\bcalcio\b/],
["iron", /\biron\b|\bhierro\b/],
["potassium", /\bpotas(?:sium|\.)?|\bpotasio\b/],
["phosphorus", /\bphosphorus\b/],
["magnesium", /\bmagnesium\b/],
["zinc", /\bzinc\b/],
["selenium", /\bselenium\b/],
["copper", /\bcopper\b/],
["manganese", /\bmanganese\b/],
["iodine", /\biodine\b/],
["chromium", /\bchromium\b/],
["molybdenum", /\bmolybdenum\b/],
["chloride", /\bchloride\b/],
["choline", /\bcholine\b/],
["caffeine", /\bcaffeine\b/],
];
// The unit printed when OCR loses it.
const defaultUnits = {
"energy-kcal": "kcal",
sodium: "mg",
cholesterol: "mg",
potassium: "mg",
calcium: "mg",
iron: "mg",
"vitamin-d": "mcg",
};
/**
* Groups boxes into reading rows. Coordinates are first turned by the
* dominant text angle, so a tilted photo still lines up "Sodium 160mg 7%".
*/
export function rows(lines) {
const angles = lines
.filter(
(line) => line.angle !== undefined && line.x1 - line.x0 > 3 * line.height,
)
.map((line) => line.angle)
.sort((a, b) => a - b);
const angle = angles.length ? angles[Math.floor(angles.length / 2)] : 0;
const cos = Math.cos(angle);
const sin = Math.sin(angle);
const placed = lines.map((line) => {
const cx = line.cx ?? (line.x0 + line.x1) / 2;
const cy = line.cy ?? (line.y0 + line.y1) / 2;
const height = line.height ?? line.y1 - line.y0;
const y = -cx * sin + cy * cos;
return {
text: line.text,
x: cx * cos + cy * sin,
y0: y - height / 2,
y1: y + height / 2,
};
});
placed.sort((a, b) => a.y0 + a.y1 - b.y0 - b.y1);
const result = [];
for (const line of placed) {
const height = line.y1 - line.y0;
const row = result.find((candidate) => {
const overlap =
Math.min(candidate.y1, line.y1) - Math.max(candidate.y0, line.y0);
return overlap > 0.5 * Math.min(height, candidate.y1 - candidate.y0);
});
if (row) row.items.push(line);
else result.push({ y0: line.y0, y1: line.y1, items: [line] });
}
return result
.sort((a, b) => a.y0 - b.y0)
.map((row) => {
row.items.sort((a, b) => a.x - b.x);
return clean(row.items.map((item) => item.text).join(" "));
});
}
// Label vocabulary; OCR misspellings within a small edit distance of these
// ("Sodlum", "Proteln", "Caicium") are snapped back before matching.
const vocabulary = `calories total fat saturated trans polyunsaturated
monounsaturated cholesterol sodium carbohydrate carbohydrates dietary fiber
soluble insoluble sugars sugar added includes alcohols protein vitamin
calcium iron potassium serving servings size container amount daily value
thiamin riboflavin niacin folate phosphorus magnesium zinc selenium copper
manganese biotin pantothenic iodine chromium molybdenum chloride choline
caffeine energy saturates fibre salt nutrition facts grasa grasas saturada
colesterol sodio fibra dietetica azucares anadidos carbohidratos hidratos
carbono proteinas calcio hierro potasio vitamina energetico contenido
porcion lipidos`.split(/\s+/);
const known = new Set(vocabulary);
function distance(a, b, limit) {
if (Math.abs(a.length - b.length) > limit) return limit + 1;
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
for (let i = 1; i <= a.length; i++) {
const current = [i];
let best = i;
for (let j = 1; j <= b.length; j++) {
current[j] = Math.min(
previous[j] + 1,
current[j - 1] + 1,
previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
);
best = Math.min(best, current[j]);
}
if (best > limit) return limit + 1;
previous = current;
}
return previous[b.length];
}
function correctWord(word) {
const lower = word.toLowerCase();
if (known.has(lower)) return word;
const limit = lower.length >= 8 ? 2 : 1;
let best = null;
let bestDistance = limit + 1;
for (const candidate of vocabulary) {
const d = distance(lower, candidate, limit);
if (d < bestDistance) {
best = candidate;
bestDistance = d;
}
}
return best ?? word;
}
/** Undoes the usual OCR confusions in label text. */
export function clean(text) {
return (
text
.normalize("NFD")
.replace(/[̀-ͯ]/g, "") // Azúcares → Azucares
.replace(/[a-z]{5,}/gi, correctWord)
.replace(/[µμ]g\b/gi, "mcg")
.replace(/\bug\b/gi, "mcg")
.replace(/less than/gi, "<")
// Letters standing in for digits next to digits or units: "Og", "1O".
.replace(/(?<=\d)[oO]|[oO](?=\d)/g, "0")
.replace(/(?<![a-z])[oO](?=\s?(?:m?c?g|%)\b)/g, "0")
.replace(/(?<=\d)[lI|](?=\d|\s?m?g\b)|(?<![a-z])[lI|](?=\d)/g, "1")
// A footnote mark read as a digit: "Carbohydrate1 13g".
.replace(/\b([a-z]{4,})[1†*]\s+(?=<?\d)/gi, "$1 ")
// Thousands separators, then decimal commas.
.replace(/(\d),(\d{3})(?!\d)/g, "$1$2")
.replace(/(\d),(\d)/g, "$1.$2")
.replace(/\s+/g, " ")
.trim()
);
}
// The amount may follow a split word ("Carbohy ydrate 42g"); a number
// directly followed by % is a percentage, not an amount.
const amountPattern =
/^\s*((?:[a-z]{1,10}\s+){0,2})[\s:.\-–—]*(<\s*)?(\d+(?:\.\d+)?|\.\d+)\s*(mcg|mg|g|kcal|kj|iu)?(?![a-z%\d.])(?!\s*%)/i;
const percentPattern = /^[^\d%]{0,12}?(\d{1,4})\s*%/;
function readAmount(rest) {
const found = amountPattern.exec(rest);
if (!found) return null;
// Never skip over another nutrient's name: in "Cholest. Protein 9g" the
// 9g is protein's.
const skipped = found[1].trim().toLowerCase().split(/\s+/).filter(Boolean);
if (skipped.some((word) => known.has(word))) return null;
const match = [found[0], ...found.slice(2)];
const after = rest.slice(match[0].length);
const percent = percentPattern.exec(after);
return {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3]?.toLowerCase(),
percent: percent ? Number(percent[1]) : null,
};
}
const toCatalog = (key, value, unit) =>
key === "energy-kcal"
? value
: convertMass(value, unit === "mcg" ? "µg" : unit, nutrientUnit(key));
/** Resolves an amount, using its %DV to correct misreads such as 8g → "89". */
function resolve(key, reading) {
const dv = dailyValues[key];
let unit = reading.unit || defaultUnits[key] || dv?.[1] || "g";
if (unit === "iu" || unit === "kj" || unit === "kcal") return null;
if (unit === "g" && dv?.[1] === "mcg") unit = "mcg"; // "mcg" read as "g"
let value = reading.less ? reading.value / 2 : reading.value;
if (dv && reading.percent !== null) {
const daily = convertMass(dv[0], mass(dv[1]), mass(unit));
// %DV is rounded (and may read 0 below 2%), so compare in points.
const consistent = (candidate) =>
Math.abs((candidate / daily) * 100 - reading.percent) <=
Math.max(2, reading.percent * 0.3);
if (daily && !consistent(value)) {
const candidates = [];
// A unit read as a digit: "8g" → "89", "160mg" → "16019".
if (!reading.unit && /\d9$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -1)));
if (!reading.unit && /\d19$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -2)));
// A lost decimal point: "05" → 0.5, "25" → 2.5.
if (/^\d{2,3}$/.test(reading.raw))
candidates.push(Number(reading.raw) / 10);
const fixed = candidates.find(consistent);
if (fixed !== undefined) value = fixed;
// Nothing plausible was read; trust the percentage.
else if (!reading.unit) value = (reading.percent / 100) * daily;
}
}
const result = toCatalog(key, value, unit);
// Far beyond a day's worth in one serving is a misread, not a food.
if (dv && result > 10 * toCatalog(key, dv[0], dv[1])) return null;
return result;
}
const mass = (unit) => (unit === "mcg" ? "µg" : unit);
function percentOnly(key, rest) {
const dv = dailyValues[key];
const match = /^[^\d]{0,6}(\d{1,4})\s*%/.exec(rest);
if (!dv || !match) return null;
return toCatalog(key, (Number(match[1]) / 100) * dv[0], dv[1]);
}
/**
* @param {{text: string, x0: number, y0: number, x1: number, y1: number}[]} lines
* @returns {{serving?: string, nutrients: Record<string, number>,
* extra: Record<string, number>, found: string[], text: string[]}}
*/
export function parseLabel(lines) {
const text = rows(lines);
const values = {};
const claimed = new Set(); // character spans already used, per row
const used = (row, start, end) => {
for (let i = start; i < end; i++)
if (claimed.has(`${row}:${i}`)) return true;
for (let i = start; i < end; i++) claimed.add(`${row}:${i}`);
return false;
};
// "Includes 10g Added Sugars" puts the amount before the name.
text.forEach((row, index) => {
const match =
/\bincl(?:udes|\.)?\s*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?\s*(?:of\s*)?added\s*sugars?/i.exec(
row,
) ||
/\b(?:added\s*sugars?|azucares\s*anadidos)[\s:]*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?/i.exec(
row,
);
if (!match || values["added-sugars"] !== undefined) return;
used(index, match.index, match.index + match[0].length);
const percent = percentPattern.exec(
row.slice(match.index + match[0].length),
);
values["added-sugars"] = resolve("added-sugars", {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3],
percent: percent ? Number(percent[1]) : null,
});
});
for (const [key, pattern] of fields) {
if (values[key] !== undefined) continue;
for (
let index = 0;
index < text.length && values[key] === undefined;
index++
) {
const regex = new RegExp(pattern.source, "gi");
for (let match; (match = regex.exec(text[index])); ) {
const start = match.index;
const end = start + match[0].length;
if (used(index, start, end)) continue;
let rest = text[index].slice(end);
// A linear label can wrap between a name and its amount.
if (
/^[\s,.:]*$/.test(rest) &&
/^<?\s*\.?\d/.test(text[index + 1] || "")
)
rest = " " + text[index + 1];
const reading = readAmount(rest);
const value = reading ? resolve(key, reading) : percentOnly(key, rest);
if (value === null || value === undefined) continue;
values[key] = value;
break;
}
}
}
if (values.salt !== undefined) {
if (values.sodium === undefined) values.sodium = (values.salt / 2.5) * 1000;
delete values.salt;
}
// Calories: the amount follows on the same row, or sits alone on a
// neighbouring row when it's printed in a much larger font.
for (let index = 0; index < text.length; index++) {
const row = text[index];
const match =
/\bcalories\b(?!\s*from)|\benergy\b|\benerg(?:ia|etico)\b/i.exec(row);
if (!match) continue;
const rest = row.slice(match.index + match[0].length);
const kcal =
/(\d+(?:\.\d+)?)\s*kcal/i.exec(rest) ||
/^[^\d]{0,24}?(\d{1,4})(?!\s*(?:k?j|%|m?c?g))\b/i.exec(rest);
const before = text[index - 1] || "";
const after = text[index + 1] || "";
const perServing =
/amount\s*per\s*serving[:\s]*(\d{1,4})\b(?!\s*(?:m?c?g|%))/i;
const alone =
// Big type often lands beside "Amount per serving" on the next row.
perServing.exec(before) ||
perServing.exec(after) ||
// A number alone at the end of a neighbouring row.
[after, before]
.map((neighbour) =>
/%|\d\s*m?c?g\b/i.test(neighbour)
? null
: /(?:^|\s)(\d{1,4})$/.exec(neighbour),
)
.find(Boolean) ||
// A linear label wrapped right after "Calories": "80, Total Fat…".
/^(\d{1,4})\b(?!\s*(?:m?c?g|%|\.\d))/.exec(after);
const value = Number((kcal || alone)?.[1]);
if (Number.isFinite(value) && value <= 5000 && (kcal || alone)) {
values["energy-kcal"] = value;
break;
}
}
let serving;
const perHundred = text.some((row) => /\bper\s*100\s*(g|ml)\b/i.test(row));
for (let index = 0; index < text.length; index++) {
const match = /\bserv(?:ing|\.)\s*size\b[\s:]*(.*)$/i.exec(text[index]);
if (!match) continue;
let size = match[1].trim();
if (!/\d/.test(size)) size = `${size} ${text[index + 1] || ""}`.trim();
// Two-column layouts can run nutrient rows into the serving text.
size = size
.split(/\b(?:calories|amount|total|sodium|protein|servings)\b|%/i)[0]
.trim()
.replace(/\s*\(\s*/g, " (")
.replace(/\s*\)\s*/g, ")")
.replace(/(\d)\s*(g|ml)\b/gi, "$1 $2")
.replace(/\)\s*.*$/, ")")
.trim();
if (size && /\d/.test(size)) serving = size.slice(0, 100);
break;
}
if (!serving && perHundred) {
const ml = text.some((row) => /\bper\s*100\s*ml\b/i.test(row));
serving = ml ? "100 ml" : "100 g";
}
const core = {
"energy-kcal": "calories",
proteins: "protein",
carbohydrates: "carbs",
fat: "fat",
fiber: "fiber",
sugars: "sugar",
sodium: "sodium",
};
const nutrients = {};
const extra = {};
for (const [key, value] of Object.entries(values)) {
if (!Number.isFinite(value) || value < 0 || value > 100000) continue;
const rounded = Math.round(value * 1000) / 1000;
if (core[key]) nutrients[core[key]] = rounded;
else extra[key] = rounded;
}
return {
...(serving ? { serving } : {}),
nutrients,
extra,
found: Object.keys(values),
text,
};
}

165
server/products/db.js Normal file
View file

@ -0,0 +1,165 @@
import { DatabaseSync } from "node:sqlite";
import { titleCase } from "../../shared/products.js";
// Each source lives in its own table so a refresh can be built alongside the
// old copy and swapped in atomically. The file is a cache: it can be deleted
// at any time and is rebuilt on the next import.
export const sources = ["usda", "off", "api"];
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
export function openProducts(filename) {
const db = new DatabaseSync(filename);
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
for (const source of ["usda", "off"])
db.exec(
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
);
// API answers include misses (nutrients NULL) so repeated scans of an
// unknown code don't hit Open Food Facts again.
db.exec(
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
);
const select = Object.fromEntries(
sources.map((source) => [
source,
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
]),
);
const toProduct = (row, source) =>
row?.nutrients
? {
gtin: row.gtin,
name: source === "usda" ? titleCase(row.name) : row.name,
brand: source === "usda" ? titleCase(row.brand) : row.brand,
serving: row.serving,
servingQty: row.serving_qty,
servingUnit: row.serving_unit,
nutrients: JSON.parse(row.nutrients),
source,
}
: null;
/** The best local match, or `undefined` when no source has an answer. */
function lookup(
gtin,
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
) {
const found = {};
for (const source of sources) found[source] = select[source].get(gtin);
const api = found.api;
if (
api &&
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
)
found.api = undefined;
const usda = toProduct(found.usda, "usda");
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
// USDA comes from manufacturers' own label submissions, so it wins;
// community-entered Open Food Facts fills in what it lacks.
const base = usda || off;
if (base)
return {
...base,
name: base.name || off?.name,
brand: base.brand || off?.brand,
};
return found.api ? null : undefined;
}
const upsertApi = db.prepare(
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
);
function remember(gtin, product) {
upsertApi.run(
gtin,
product?.name ?? null,
product?.brand ?? null,
product?.serving ?? null,
product?.servingQty ?? null,
product?.servingUnit ?? null,
product ? JSON.stringify(product.nutrients) : null,
Date.now(),
);
}
/** Bulk loader used by the importers: writes `<source>_products_next`. */
function beginImport(source) {
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
const insert = db.prepare(
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
);
let pending = 0;
let rows = 0;
db.exec("BEGIN");
return {
add(product) {
insert.run(
product.gtin,
product.name || null,
product.brand || null,
product.serving || null,
product.servingQty ?? null,
product.servingUnit || null,
JSON.stringify(product.nutrients),
);
rows++;
if (++pending >= 20000) {
db.exec("COMMIT; BEGIN");
pending = 0;
}
},
commit(version) {
db.exec(`COMMIT; BEGIN IMMEDIATE;
DROP TABLE IF EXISTS ${source}_products;
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
db.prepare(
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
VALUES (?,?,?,?,?,NULL)`,
).run(
source,
version,
new Date().toISOString(),
rows,
new Date().toISOString(),
);
db.exec("COMMIT");
// Old pages are reused by the next import; reclaiming them would
// rewrite the whole file.
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
return rows;
},
abort() {
if (db.isTransaction) db.exec("ROLLBACK");
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
},
};
}
const state = (source) =>
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
function recordCheck(source, error = null) {
db.prepare(
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
).run(source, new Date().toISOString(), error);
}
const status = () =>
db
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
.all();
return {
db,
lookup,
remember,
beginImport,
state,
recordCheck,
status,
close: () => db.close(),
};
}

94
server/products/import.js Normal file
View file

@ -0,0 +1,94 @@
// Refreshes the local product databases. Runs as a low-priority child process
// (see scheduler.js) so parsing several gigabytes never stalls the app.
// node server/products/import.js <data dir> [--force]
import { mkdirSync, rmSync } from "node:fs";
import { setPriority } from "node:os";
import { join, resolve } from "node:path";
import { openProducts } from "./db.js";
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
const dataDir = resolve(process.argv[2] || "data");
const force = process.argv.includes("--force");
const OFF_MAX_AGE = 7 * 86400000;
const log = (...args) => console.log("[products]", ...args);
try {
setPriority(19);
} catch {
// Lowering priority is best effort.
}
const products = openProducts(join(dataDir, "products.sqlite"));
const tmp = join(dataDir, "tmp");
mkdirSync(tmp, { recursive: true });
async function refreshUsda() {
const latest = await latestUsdaArchive();
if (!force && products.state("usda")?.version === latest.version)
return products.recordCheck("usda");
const file = join(tmp, "usda-branded.zip");
log(`downloading USDA Branded Foods ${latest.version}`);
try {
await download(latest.url, file);
const sink = products.beginImport("usda");
try {
await importUsdaZip(file, sink, {
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
});
log(
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
} finally {
rmSync(file, { force: true });
}
}
async function refreshOff() {
const state = products.state("off");
if (
!force &&
state?.imported_at &&
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
)
return;
const response = await fetch(OFF_CSV, {
headers: { "User-Agent": userAgent },
});
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const version =
response.headers.get("last-modified") || new Date().toUTCString();
if (!force && state?.version === version) {
await response.body?.cancel();
return products.recordCheck("off");
}
log("streaming the Open Food Facts export");
const sink = products.beginImport("off");
try {
await importOffCsv(response, sink, {
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
});
log(
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
);
} catch (error) {
sink.abort();
throw error;
}
}
for (const [source, refresh] of [
["usda", refreshUsda],
["off", refreshOff],
]) {
try {
await refresh();
} catch (error) {
console.error(`[products] ${source} refresh failed:`, error.message);
products.recordCheck(source, String(error.message).slice(0, 300));
}
}
products.close();

175
server/products/off.js Normal file
View file

@ -0,0 +1,175 @@
import { createGunzip } from "node:zlib";
import { Readable } from "node:stream";
import { databaseGtin } from "../../shared/barcode.js";
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
export const OFF_CSV =
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
export const userAgent =
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
// Columns that are not nutrient amounts, or duplicate one handled separately.
const skipped = new Set([
"energy",
"energy-kj",
"energy-from-fat",
"salt",
"added-salt",
"ph",
"alcohol", // % vol in Open Food Facts, not grams
"cocoa",
"chlorophyl",
"collagen-meat-protein-ratio",
"glycemic-index",
"water-hardness",
"acidity",
"carbohydrates-total",
]);
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
/**
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
* stored units.
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
*/
export function offNutrients(entries) {
const raw = new Map();
for (const [key, value] of entries) {
const number = typeof value === "number" ? value : Number(value);
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
continue;
raw.set(key, number);
}
const nutrients = {};
let kcal = raw.get("energy-kcal");
if (kcal === undefined) {
const kj = raw.get("energy-kj") ?? raw.get("energy");
if (kj !== undefined) kcal = kj / 4.184;
}
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
let sodium = raw.get("sodium");
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
for (const [key, grams] of raw) {
if (
key === "energy-kcal" ||
key === "sodium" ||
skipped.has(key) ||
notNutrient.test(key) ||
!nutrientKeyPattern.test(key) ||
grams > 100 // more than 100 g per 100 g is a data-entry error
)
continue;
const value = fromGrams(key, grams);
if (value !== null) nutrients[key] = value;
}
return nutrients;
}
function servingUnit(text) {
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
? "ml"
: "g";
}
/** Shapes a CSV row or API product into a products-table row. */
export function offProduct({
code,
name,
brands,
serving,
servingQty,
nutrients,
}) {
const gtin = databaseGtin(code);
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
const qty = Number(servingQty);
return {
gtin,
name: (name || "").trim().slice(0, 200) || null,
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
serving: (serving || "").trim().slice(0, 100) || null,
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
servingUnit: servingUnit(serving),
nutrients,
};
}
async function* lines(stream) {
let rest = "";
stream.setEncoding("utf8");
for await (const chunk of stream) {
const parts = (rest + chunk).split("\n");
rest = parts.pop();
yield* parts;
}
if (rest) yield rest;
}
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
export async function importOffCsv(response, sink, { onProgress } = {}) {
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
let header;
let count = 0;
for await (const line of lines(gunzip)) {
const fields = line.split("\t");
if (!header) {
header = Object.fromEntries(fields.map((name, i) => [name, i]));
for (const column of ["code", "product_name", "energy-kcal_100g"])
if (!(column in header))
throw new Error(`Open Food Facts export is missing ${column}`);
header.nutrientColumns = fields
.map((name, i) => [name, i])
.filter(([name]) => name.endsWith("_100g"))
.map(([name, i]) => [name.slice(0, -5), i]);
continue;
}
if (fields[header.no_nutrition_data] === "on") continue;
const product = offProduct({
code: fields[header.code],
name: fields[header.product_name],
brands: fields[header.brands],
serving: fields[header.serving_size],
servingQty: fields[header.serving_quantity],
nutrients: offNutrients(
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
),
});
if (!product) continue;
sink.add(product);
if (++count % 250000 === 0) onProgress?.(count);
}
if (!header) throw new Error("Open Food Facts export was empty");
return count;
}
const apiFields =
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
/** Live lookup, used only when no local copy has the product. */
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
// Open Food Facts keys products by EAN-13 (or EAN-8).
const stripped = gtin.replace(/^0+/, "");
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
const response = await fetchImpl(
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
);
if (response.status === 404) return null;
if (!response.ok)
throw new Error(`Open Food Facts answered ${response.status}`);
const data = await response.json();
if (data.status !== 1 || !data.product) return null;
const p = data.product;
const entries = Object.entries(p.nutriments || {})
.filter(([key]) => key.endsWith("_100g"))
.map(([key, value]) => [key.slice(0, -5), value]);
const product = offProduct({
code: gtin,
name: p.product_name_en || p.product_name,
brands: p.brands,
serving: p.serving_size,
servingQty: p.serving_quantity,
nutrients: offNutrients(entries),
});
return product && { ...product, source: "api" };
}

View file

@ -0,0 +1,27 @@
import { fork } from "node:child_process";
import { fileURLToPath } from "node:url";
const script = fileURLToPath(new URL("./import.js", import.meta.url));
const HOUR = 3600000;
/**
* Checks for new product data shortly after start-up and every six hours.
* The import itself decides whether anything needs downloading.
*/
export function scheduleProductImports(dataDir, { firstDelay = 60000 } = {}) {
let child = null;
const run = () => {
if (child) return;
child = fork(script, [dataDir], {
execArgv: ["--max-old-space-size=1024"],
stdio: "inherit",
});
child.on("exit", () => (child = null));
};
const timers = [setTimeout(run, firstDelay), setInterval(run, 6 * HOUR)];
for (const timer of timers) timer.unref();
return () => {
for (const timer of timers) clearInterval(timer);
child?.kill();
};
}

398
server/products/usda.js Normal file
View file

@ -0,0 +1,398 @@
import { createWriteStream } from "node:fs";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import yauzl from "yauzl";
import { databaseGtin } from "../../shared/barcode.js";
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
const archiveName =
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
/** The newest Branded Foods CSV archive linked from the download page. */
export async function latestUsdaArchive(fetchImpl = fetch) {
const response = await fetchImpl(USDA_PAGE, {
signal: AbortSignal.timeout(30000),
});
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
const versions = [...(await response.text()).matchAll(archiveName)].map(
(match) => match[1],
);
if (!versions.length) throw new Error("No USDA branded archive is listed");
const version = versions.sort().at(-1);
return {
version,
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
};
}
export async function download(url, file, fetchImpl = fetch) {
const response = await fetchImpl(url);
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
}
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
// are kept under a slug of their USDA name.
const nutrientIds = {
1008: "energy-kcal",
1003: "proteins",
1004: "fat",
1005: "carbohydrates",
1079: "fiber",
2000: "sugars",
1093: "sodium",
1258: "saturated-fat",
1257: "trans-fat",
1292: "monounsaturated-fat",
1293: "polyunsaturated-fat",
1253: "cholesterol",
1235: "added-sugars",
1086: "polyols",
1082: "soluble-fiber",
1084: "insoluble-fiber",
1009: "starch",
1010: "sucrose",
1011: "glucose",
1012: "fructose",
1013: "lactose",
1014: "maltose",
1075: "galactose",
1056: "sorbitol",
1078: "xylitol",
1055: "mannitol",
1051: "water",
1018: "alcohol",
1057: "caffeine",
1234: "taurine",
1068: "beta-glucan",
1181: "inositol",
1106: "vitamin-a",
1104: "vitamin-a-iu",
1162: "vitamin-c",
1114: "vitamin-d",
1109: "vitamin-e",
1185: "vitamin-k",
1165: "vitamin-b1",
1166: "vitamin-b2",
1167: "vitamin-pp",
1175: "vitamin-b6",
1190: "vitamin-b9",
1177: "folates",
1186: "folic-acid",
1178: "vitamin-b12",
1176: "biotin",
1170: "pantothenic-acid",
1180: "choline",
1087: "calcium",
1089: "iron",
1092: "potassium",
1090: "magnesium",
1091: "phosphorus",
1095: "zinc",
1098: "copper",
1101: "manganese",
1103: "selenium",
1100: "iodine",
1096: "chromium",
1102: "molybdenum",
1088: "chloride",
1099: "fluoride",
};
// Fallbacks used only when the preferred ID is absent.
const fallbacks = {
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
1085: ["fat", (g) => g], // Total fat (NLEA)
1063: ["sugars", (g) => g], // Sugars, Total
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
};
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
const slug = (name) =>
name
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, "")
.slice(0, 48);
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
export function nutrientMapper(nutrientRows) {
const map = new Map();
for (const { id, name, unit } of nutrientRows) {
const from = usdaUnits[unit];
let key = nutrientIds[id];
if (fallbacks[id]) {
const [target, convert] = fallbacks[id];
map.set(id, { key: target, convert, fallback: true });
continue;
}
if (!key) {
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
continue;
key = slug(name);
if (!key) continue;
}
const to = nutrientUnit(key);
const convert =
from === to ? (value) => value : (value) => convertMass(value, from, to);
if (from !== to && convertMass(1, from, to) === null) continue;
map.set(id, { key, convert, fallback: false });
}
return map;
}
// Parses one record starting at `pos`; null when `text` ends mid-record.
function parseRecord(text, pos) {
const fields = [];
for (;;) {
if (pos >= text.length) return null;
if (text[pos] === '"') {
let value = "";
let i = pos + 1;
for (;;) {
const quote = text.indexOf('"', i);
if (quote === -1 || quote + 1 >= text.length) return null;
value += text.slice(i, quote);
if (text[quote + 1] !== '"') {
pos = quote + 1;
break;
}
value += '"'; // "" inside a quoted field
i = quote + 2;
}
fields.push(value);
} else {
let end = pos;
while (end < text.length && text[end] !== "," && text[end] !== "\n")
end++;
if (end >= text.length) return null;
fields.push(text.slice(pos, end).replace(/\r$/, ""));
pos = end;
}
const c = text[pos];
if (c === ",") pos++;
else if (c === "\n") return { fields, next: pos + 1 };
else if (c === "\r" && text[pos + 1] === "\n")
return { fields, next: pos + 2 };
else if (pos + 1 >= text.length) return null;
else pos++; // stray character after a closing quote
}
}
/** Minimal RFC 4180 reader: yields one array of fields per record. */
export async function* csvRecords(stream) {
stream.setEncoding("utf8");
let buffer = "";
for await (const chunk of stream) {
buffer += chunk;
let start = 0;
for (let record; (record = parseRecord(buffer, start)); start = record.next)
yield record.fields;
buffer = buffer.slice(start);
}
if (buffer.trim()) {
const record = parseRecord(buffer + "\n", 0);
if (record) yield record.fields;
}
}
function openZip(file) {
return new Promise((resolve, reject) =>
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
if (error) return reject(error);
const entries = new Map();
zip.on("entry", (entry) =>
entries.set(entry.fileName.split("/").at(-1), entry),
);
zip.on("end", () => resolve({ zip, entries }));
zip.on("error", reject);
}),
);
}
// Yields the requested columns of each row, in the order asked for.
async function* table(zip, entries, name, columns) {
const entry = entries.get(name);
if (!entry) throw new Error(`USDA archive is missing ${name}`);
const stream = await new Promise((resolve, reject) =>
zip.openReadStream(entry, (error, s) =>
error ? reject(error) : resolve(s),
),
);
let indexes;
for await (const fields of csvRecords(stream)) {
if (!indexes) {
indexes = columns.map((column) => fields.indexOf(column));
const missing = columns.filter((_, i) => indexes[i] < 0);
if (missing.length)
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
continue;
}
yield indexes.map((i) => fields[i]);
}
}
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
function servingText(household, qty, unit) {
const amount = `${+qty.toFixed(1)} ${unit}`;
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
return amount;
// Some submissions already include the weight: "3/4 cup (20g)".
return /\d\s*(g|ml|grams?)\b/i.test(household)
? household
: `${household} (${amount})`;
}
/** Reads a Branded Foods CSV archive into `sink`. */
export async function importUsdaZip(file, sink, { onProgress } = {}) {
const { zip, entries } = await openZip(file);
try {
const nutrientRows = [];
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
"id",
"name",
"unit_name",
]))
nutrientRows.push({ id: Number(id), name, unit });
const mapper = nutrientMapper(nutrientRows);
// A GTIN can appear several times as a product is resubmitted; keep the
// newest submission.
const byGtin = new Map();
const strings = new Map();
// Substrings keep their whole parse buffer alive; store fresh copies,
// shared across the many products with the same brand or serving.
const keep = (text) => {
if (!text) return "";
let copy = strings.get(text);
if (copy === undefined) {
copy = Buffer.from(text).toString();
strings.set(copy, copy);
}
return copy;
};
for await (const [
id,
gtinUpc,
brandName,
brandOwner,
servingSize,
servingSizeUnit,
householdText,
available,
modified,
] of table(zip, entries, "branded_food.csv", [
"fdc_id",
"gtin_upc",
"brand_name",
"brand_owner",
"serving_size",
"serving_size_unit",
"household_serving_fulltext",
"available_date",
"modified_date",
])) {
const gtin = databaseGtin(gtinUpc);
if (!gtin) continue;
const date = available || modified || "";
const fdcId = Number(id);
const current = byGtin.get(gtin);
if (
current &&
(current.date > date ||
(current.date === date && current.fdcId > fdcId))
)
continue;
const qty = Number(servingSize);
const household = (householdText || "").trim();
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
byGtin.set(keep(gtin), {
fdcId,
date: keep(date),
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
servingQty: unit && qty > 0 ? qty : null,
servingUnit: unit || "g",
serving:
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
name: "",
gtin: "",
});
}
strings.clear();
const byFdc = new Map();
for (const [gtin, product] of byGtin) {
product.gtin = gtin;
byFdc.set(product.fdcId, product);
}
byGtin.clear();
for await (const [id, description] of table(zip, entries, "food.csv", [
"fdc_id",
"description",
])) {
const product = byFdc.get(Number(id));
if (product)
product.name = Buffer.from(description.slice(0, 200)).toString();
}
let count = 0;
const finish = (product, values) => {
const nutrients = {};
const late = [];
for (let i = 0; i < values.length; i += 2) {
const mapping = mapper.get(values[i]);
const number = values[i + 1];
if (!mapping || !Number.isFinite(number) || number < 0) continue;
if (mapping.fallback) late.push([mapping, number]);
else {
const value = mapping.convert(number);
if (value !== null) nutrients[mapping.key] = value;
}
}
for (const [mapping, number] of late)
if (nutrients[mapping.key] === undefined)
nutrients[mapping.key] = mapping.convert(number);
if (
nutrients["energy-kcal"] === undefined ||
nutrients["energy-kcal"] > 950
)
return;
sink.add({
gtin: product.gtin,
name: product.name,
brand: product.brand,
serving: product.serving,
servingQty: product.servingQty,
servingUnit: product.servingUnit,
nutrients,
});
if (++count % 250000 === 0) onProgress?.(count);
};
// ~30 million rows, grouped by food: each product is finished as soon as
// the next one starts, so only one food's nutrients are held at a time.
let currentId = null;
let values = [];
for await (const [id, nutrientId, amount] of table(
zip,
entries,
"food_nutrient.csv",
["fdc_id", "nutrient_id", "amount"],
)) {
const fdcId = Number(id);
if (fdcId !== currentId) {
const done = byFdc.get(currentId);
if (done) {
finish(done, values);
byFdc.delete(currentId);
}
currentId = fdcId;
values = [];
}
if (byFdc.has(fdcId) && amount !== "")
values.push(Number(nutrientId), Number(amount));
}
const last = byFdc.get(currentId);
if (last) finish(last, values);
return count;
} finally {
zip.close();
}
}