Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
8f6981c8a2
commit
80fc48b9f1
39 changed files with 4434 additions and 33 deletions
|
|
@ -6,7 +6,11 @@ import { resolve, join } from "node:path";
|
|||
import { existsSync } from "node:fs";
|
||||
import { z } from "zod";
|
||||
import { recordSchema } from "../shared/domain.js";
|
||||
import { normalizeBarcode } from "../shared/barcode.js";
|
||||
import { productToFood } from "../shared/products.js";
|
||||
import { checkPassword, tokenHash } from "./auth.js";
|
||||
import { fetchOffProduct } from "./products/off.js";
|
||||
import { parseLabel } from "./ocr/label.js";
|
||||
|
||||
export function createApp({
|
||||
store,
|
||||
|
|
@ -15,6 +19,9 @@ export function createApp({
|
|||
origin,
|
||||
secureCookies = true,
|
||||
dist = resolve("dist"),
|
||||
products,
|
||||
fetchProduct = fetchOffProduct,
|
||||
ocr,
|
||||
}) {
|
||||
const app = express();
|
||||
app.disable("x-powered-by");
|
||||
|
|
@ -22,9 +29,10 @@ export function createApp({
|
|||
helmet({
|
||||
contentSecurityPolicy: {
|
||||
directives: {
|
||||
"script-src": ["'self'"],
|
||||
// The barcode reader is WebAssembly, served from this origin.
|
||||
"script-src": ["'self'", "'wasm-unsafe-eval'"],
|
||||
"style-src": ["'self'", "'unsafe-inline'"],
|
||||
"img-src": ["'self'", "data:"],
|
||||
"img-src": ["'self'", "data:", "blob:"],
|
||||
"connect-src": ["'self'"],
|
||||
"object-src": ["'none'"],
|
||||
"upgrade-insecure-requests": secureCookies ? [] : null,
|
||||
|
|
@ -146,6 +154,77 @@ export function createApp({
|
|||
});
|
||||
}
|
||||
});
|
||||
app.get("/api/products/:code", requireAuth, async (req, res) => {
|
||||
const gtin = normalizeBarcode(req.params.code);
|
||||
if (!gtin)
|
||||
return res
|
||||
.status(400)
|
||||
.json({ error: "That isn’t a valid product barcode." });
|
||||
if (!products)
|
||||
return res
|
||||
.status(503)
|
||||
.json({ error: "The product database isn’t available." });
|
||||
let product = products.lookup(gtin);
|
||||
if (product === undefined) {
|
||||
try {
|
||||
product = await fetchProduct(gtin);
|
||||
} catch {
|
||||
return res.status(502).json({
|
||||
error:
|
||||
"This product isn’t in the local database yet, and Open Food Facts couldn’t be reached.",
|
||||
gtin,
|
||||
});
|
||||
}
|
||||
products.remember(gtin, product);
|
||||
}
|
||||
if (!product)
|
||||
return res
|
||||
.status(404)
|
||||
.json({ error: "No nutrition information for this barcode.", gtin });
|
||||
res.json({ food: productToFood(product) });
|
||||
});
|
||||
app.get("/api/products", requireAuth, (_, res) =>
|
||||
res.json({ sources: products?.status() || [] }),
|
||||
);
|
||||
// One label at a time: recognition already uses every core.
|
||||
let labelQueue = Promise.resolve();
|
||||
app.post(
|
||||
"/api/label",
|
||||
requireAuth,
|
||||
express.raw({
|
||||
type: ["image/jpeg", "image/png", "image/webp"],
|
||||
limit: "16mb",
|
||||
}),
|
||||
async (req, res) => {
|
||||
if (!Buffer.isBuffer(req.body) || !req.body.length)
|
||||
return res
|
||||
.status(415)
|
||||
.json({ error: "Send a JPEG, PNG or WebP photo of the label." });
|
||||
if (!ocr)
|
||||
return res
|
||||
.status(503)
|
||||
.json({ error: "Label scanning isn’t available on this server." });
|
||||
const run = labelQueue.then(async () => parseLabel(await ocr(req.body)));
|
||||
labelQueue = run.catch(() => {});
|
||||
try {
|
||||
const label = await run;
|
||||
if (!label.found.length)
|
||||
return res.status(422).json({
|
||||
error:
|
||||
"No nutrition facts were found. Fill the frame with the label and keep it sharp.",
|
||||
});
|
||||
res.json({ label });
|
||||
} catch (error) {
|
||||
if (error?.unavailable)
|
||||
return res
|
||||
.status(503)
|
||||
.json({ error: "Label scanning isn’t available on this server." });
|
||||
res
|
||||
.status(422)
|
||||
.json({ error: "That photo couldn’t be read. Try another one." });
|
||||
}
|
||||
},
|
||||
);
|
||||
app.use("/api", (_, res) => res.status(404).json({ error: "Not found" }));
|
||||
if (existsSync(dist)) {
|
||||
// The service worker is also served without a file extension. Reverse
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@ import { resolve } from "node:path";
|
|||
import { openDatabase } from "./db.js";
|
||||
import { createApp } from "./app.js";
|
||||
import { hashPassword } from "./auth.js";
|
||||
import { openProducts } from "./products/db.js";
|
||||
import { scheduleProductImports } from "./products/scheduler.js";
|
||||
import { createOcr } from "./ocr/engine.js";
|
||||
const secret = (name) =>
|
||||
process.env[name + "_FILE"]
|
||||
? readFileSync(process.env[name + "_FILE"], "utf8").trimEnd()
|
||||
|
|
@ -30,12 +33,30 @@ if (
|
|||
const dataDir = resolve(process.env.DATA_DIR || "data");
|
||||
mkdirSync(dataDir, { recursive: true });
|
||||
const store = openDatabase(resolve(dataDir, "calio.sqlite"));
|
||||
const products = openProducts(resolve(dataDir, "products.sqlite"));
|
||||
const stopImports = scheduleProductImports(dataDir);
|
||||
// Models load in the background so the first label scan doesn't wait.
|
||||
const engine = createOcr(resolve(process.env.MODELS_DIR || "models")).catch(
|
||||
(error) => {
|
||||
console.error("Label scanning is unavailable:", error.message);
|
||||
return null;
|
||||
},
|
||||
);
|
||||
const app = createApp({
|
||||
store,
|
||||
username: process.env.CALIO_USERNAME || "calio",
|
||||
passwordHash,
|
||||
origin,
|
||||
secureCookies,
|
||||
products,
|
||||
ocr: async (image) => {
|
||||
const ocr = await engine;
|
||||
if (!ocr)
|
||||
throw Object.assign(new Error("OCR models are not loaded"), {
|
||||
unavailable: true,
|
||||
});
|
||||
return ocr.read(image);
|
||||
},
|
||||
});
|
||||
const server = app.listen(Number(process.env.PORT || 3000), "0.0.0.0", () =>
|
||||
console.log(`Calio listening at ${origin}`),
|
||||
|
|
@ -43,6 +64,8 @@ const server = app.listen(Number(process.env.PORT || 3000), "0.0.0.0", () =>
|
|||
for (const signal of ["SIGTERM", "SIGINT"])
|
||||
process.on(signal, () =>
|
||||
server.close(() => {
|
||||
stopImports();
|
||||
products.close();
|
||||
store.close();
|
||||
process.exit(0);
|
||||
}),
|
||||
|
|
|
|||
434
server/ocr/engine.js
Normal file
434
server/ocr/engine.js
Normal file
|
|
@ -0,0 +1,434 @@
|
|||
// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors
|
||||
// PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with
|
||||
// unclip, then CTC decoding of each line crop.
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import ort from "onnxruntime-node";
|
||||
import sharp from "sharp";
|
||||
|
||||
const DET_THRESH = 0.3;
|
||||
const BOX_THRESH = 0.6;
|
||||
const UNCLIP_RATIO = 1.5;
|
||||
const REC_HEIGHT = 48;
|
||||
const REC_BATCH = 8;
|
||||
|
||||
/** Reads the CTC character list out of a PaddleOCR inference.yml. */
|
||||
export function readDictionary(yaml) {
|
||||
const lines = yaml.split(/\r?\n/);
|
||||
const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line));
|
||||
if (start < 0) throw new Error("No character_dict in model config");
|
||||
const chars = [];
|
||||
for (const line of lines.slice(start + 1)) {
|
||||
const match = /^\s*- (.*)$/.exec(line);
|
||||
if (!match) break;
|
||||
let value = match[1];
|
||||
if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'");
|
||||
else if (value.startsWith('"')) value = JSON.parse(value);
|
||||
chars.push(value);
|
||||
}
|
||||
return ["", ...chars, " "]; // blank, dictionary, then the space character
|
||||
}
|
||||
|
||||
// ---------- geometry ----------
|
||||
|
||||
function convexHull(points) {
|
||||
points.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
||||
const cross = (o, a, b) =>
|
||||
(a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]);
|
||||
const lower = [];
|
||||
for (const p of points) {
|
||||
while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0)
|
||||
lower.pop();
|
||||
lower.push(p);
|
||||
}
|
||||
const upper = [];
|
||||
for (let i = points.length - 1; i >= 0; i--) {
|
||||
const p = points[i];
|
||||
while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0)
|
||||
upper.pop();
|
||||
upper.push(p);
|
||||
}
|
||||
return lower.slice(0, -1).concat(upper.slice(0, -1));
|
||||
}
|
||||
|
||||
/** Minimum-area rectangle as centre, unit axes and side lengths. */
|
||||
export function minAreaRect(points) {
|
||||
const hull = convexHull(points);
|
||||
if (hull.length < 3) {
|
||||
const [x, y] = hull[0] || [0, 0];
|
||||
return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 };
|
||||
}
|
||||
let best = null;
|
||||
for (let i = 0; i < hull.length; i++) {
|
||||
const [x1, y1] = hull[i];
|
||||
const [x2, y2] = hull[(i + 1) % hull.length];
|
||||
const length = Math.hypot(x2 - x1, y2 - y1);
|
||||
if (!length) continue;
|
||||
const ux = (x2 - x1) / length;
|
||||
const uy = (y2 - y1) / length;
|
||||
let minU = Infinity;
|
||||
let maxU = -Infinity;
|
||||
let minV = Infinity;
|
||||
let maxV = -Infinity;
|
||||
for (const [x, y] of hull) {
|
||||
const u = x * ux + y * uy;
|
||||
const v = -x * uy + y * ux;
|
||||
minU = Math.min(minU, u);
|
||||
maxU = Math.max(maxU, u);
|
||||
minV = Math.min(minV, v);
|
||||
maxV = Math.max(maxV, v);
|
||||
}
|
||||
const area = (maxU - minU) * (maxV - minV);
|
||||
if (!best || area < best.area) {
|
||||
const cu = (minU + maxU) / 2;
|
||||
const cv = (minV + maxV) / 2;
|
||||
best = {
|
||||
area,
|
||||
cx: cu * ux - cv * uy,
|
||||
cy: cu * uy + cv * ux,
|
||||
ux,
|
||||
uy,
|
||||
w: maxU - minU,
|
||||
h: maxV - minV,
|
||||
};
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orients a rectangle for reading: the axis nearer horizontal runs left to
|
||||
* right, the other top to bottom. Returns the top-left corner and axes.
|
||||
*/
|
||||
function readingFrame({ cx, cy, ux, uy, w, h }) {
|
||||
let ax = [ux, uy, w];
|
||||
let bx = [-uy, ux, h];
|
||||
if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax];
|
||||
if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]];
|
||||
if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]];
|
||||
const [dx, dy, width] = ax;
|
||||
const [ex, ey, height] = bx;
|
||||
return {
|
||||
x: cx - (dx * width) / 2 - (ex * height) / 2,
|
||||
y: cy - (dy * width) / 2 - (ey * height) / 2,
|
||||
dx,
|
||||
dy,
|
||||
ex,
|
||||
ey,
|
||||
width,
|
||||
height,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- detection ----------
|
||||
|
||||
function detectionBoxes(prob, width, height) {
|
||||
const visited = new Uint8Array(width * height);
|
||||
const boxes = [];
|
||||
const stack = [];
|
||||
for (let start = 0; start < prob.length; start++) {
|
||||
if (visited[start] || prob[start] <= DET_THRESH) continue;
|
||||
// Flood-fill one connected region, keeping each row's extreme pixels as
|
||||
// hull candidates.
|
||||
const rows = new Map();
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
stack.push(start);
|
||||
visited[start] = 1;
|
||||
while (stack.length) {
|
||||
const index = stack.pop();
|
||||
const x = index % width;
|
||||
const y = (index - x) / width;
|
||||
sum += prob[index];
|
||||
count++;
|
||||
const row = rows.get(y);
|
||||
if (!row) rows.set(y, [x, x]);
|
||||
else {
|
||||
if (x < row[0]) row[0] = x;
|
||||
if (x > row[1]) row[1] = x;
|
||||
}
|
||||
for (const next of [
|
||||
x > 0 ? index - 1 : -1,
|
||||
x < width - 1 ? index + 1 : -1,
|
||||
y > 0 ? index - width : -1,
|
||||
y < height - 1 ? index + width : -1,
|
||||
])
|
||||
if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) {
|
||||
visited[next] = 1;
|
||||
stack.push(next);
|
||||
}
|
||||
}
|
||||
if (count < 4 || sum / count < BOX_THRESH) continue;
|
||||
const points = [];
|
||||
for (const [y, [x0, x1]] of rows)
|
||||
points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]);
|
||||
const rect = minAreaRect(points);
|
||||
if (Math.min(rect.w, rect.h) < 3) continue;
|
||||
// Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper.
|
||||
const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h));
|
||||
rect.w += 2 * d;
|
||||
rect.h += 2 * d;
|
||||
if (Math.min(rect.w, rect.h) < 5) continue;
|
||||
boxes.push({ ...rect, score: sum / count });
|
||||
}
|
||||
return boxes;
|
||||
}
|
||||
|
||||
// ---------- recognition ----------
|
||||
|
||||
/** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */
|
||||
function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) {
|
||||
const { data, width, height } = image;
|
||||
const plane = REC_HEIGHT * batchWidth;
|
||||
const scale = frame.height / REC_HEIGHT;
|
||||
const taps = Math.min(4, Math.max(1, Math.round(scale)));
|
||||
const pixel = (x, y, c) => {
|
||||
const xi = Math.min(width - 1, Math.max(0, x));
|
||||
const yi = Math.min(height - 1, Math.max(0, y));
|
||||
return data[(yi * width + xi) * 3 + c];
|
||||
};
|
||||
for (let ty = 0; ty < REC_HEIGHT; ty++)
|
||||
for (let tx = 0; tx < cropWidth; tx++) {
|
||||
let r = 0;
|
||||
let g = 0;
|
||||
let b = 0;
|
||||
for (let sy = 0; sy < taps; sy++)
|
||||
for (let sx = 0; sx < taps; sx++) {
|
||||
const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width;
|
||||
const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height;
|
||||
const x = frame.x + frame.dx * u + frame.ex * v - 0.5;
|
||||
const y = frame.y + frame.dy * u + frame.ey * v - 0.5;
|
||||
const x0 = Math.floor(x);
|
||||
const y0 = Math.floor(y);
|
||||
const fx = x - x0;
|
||||
const fy = y - y0;
|
||||
for (let c = 0; c < 3; c++) {
|
||||
const value =
|
||||
pixel(x0, y0, c) * (1 - fx) * (1 - fy) +
|
||||
pixel(x0 + 1, y0, c) * fx * (1 - fy) +
|
||||
pixel(x0, y0 + 1, c) * (1 - fx) * fy +
|
||||
pixel(x0 + 1, y0 + 1, c) * fx * fy;
|
||||
if (c === 0) r += value;
|
||||
else if (c === 1) g += value;
|
||||
else b += value;
|
||||
}
|
||||
}
|
||||
const n = taps * taps * 127.5;
|
||||
const at = offset + ty * batchWidth + tx;
|
||||
out[at] = b / n - 1;
|
||||
out[at + plane] = g / n - 1;
|
||||
out[at + 2 * plane] = r / n - 1;
|
||||
}
|
||||
}
|
||||
|
||||
function ctcDecode(output, dims, chars) {
|
||||
const [count, steps, classes] = dims;
|
||||
const results = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
let text = "";
|
||||
let confidence = 0;
|
||||
let kept = 0;
|
||||
let previous = -1;
|
||||
for (let t = 0; t < steps; t++) {
|
||||
const base = (n * steps + t) * classes;
|
||||
let best = 0;
|
||||
let bestValue = output[base];
|
||||
for (let c = 1; c < classes; c++)
|
||||
if (output[base + c] > bestValue) {
|
||||
best = c;
|
||||
bestValue = output[base + c];
|
||||
}
|
||||
if (best !== 0 && best !== previous) {
|
||||
text += chars[best] ?? "";
|
||||
confidence += bestValue;
|
||||
kept++;
|
||||
}
|
||||
previous = best;
|
||||
}
|
||||
results.push({ text, score: kept ? confidence / kept : 0 });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ---------- pipeline ----------
|
||||
|
||||
export async function createOcr(modelDir, { detSide = 1600 } = {}) {
|
||||
const options = {
|
||||
graphOptimizationLevel: "all",
|
||||
// The arena would keep ~2 GB of detection buffers after the first scan.
|
||||
enableCpuMemArena: false,
|
||||
};
|
||||
const [det, rec] = await Promise.all([
|
||||
ort.InferenceSession.create(
|
||||
join(modelDir, "det", "inference.onnx"),
|
||||
options,
|
||||
),
|
||||
ort.InferenceSession.create(
|
||||
join(modelDir, "rec", "inference.onnx"),
|
||||
options,
|
||||
),
|
||||
]);
|
||||
const chars = readDictionary(
|
||||
readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"),
|
||||
);
|
||||
|
||||
async function detect(image) {
|
||||
// Small photos are enlarged too: tiny print detects far better.
|
||||
const ratio = Math.min(2, detSide / Math.max(image.width, image.height));
|
||||
const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32);
|
||||
const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
|
||||
const { data } = await sharp(image.data, {
|
||||
raw: { width: image.width, height: image.height, channels: 3 },
|
||||
})
|
||||
.resize(w, h, { fit: "fill" })
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
const input = new Float32Array(3 * w * h);
|
||||
const mean = [0.406, 0.456, 0.485]; // applied to B, G, R
|
||||
const std = [0.225, 0.224, 0.229];
|
||||
for (let i = 0; i < w * h; i++)
|
||||
for (let c = 0; c < 3; c++)
|
||||
// Channel c of the tensor is BGR; data is RGB.
|
||||
input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c];
|
||||
const result = await det.run({
|
||||
[det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]),
|
||||
});
|
||||
const prob = result[det.outputNames[0]].data;
|
||||
const sx = image.width / w;
|
||||
const sy = image.height / h;
|
||||
return detectionBoxes(prob, w, h).map((box) => {
|
||||
// Scale the rectangle back to the full image.
|
||||
const ax = box.ux * sx;
|
||||
const ay = box.uy * sy;
|
||||
const length = Math.hypot(ax, ay);
|
||||
const bx = -box.uy * sx;
|
||||
const by = box.ux * sy;
|
||||
return {
|
||||
cx: box.cx * sx,
|
||||
cy: box.cy * sy,
|
||||
ux: ax / length,
|
||||
uy: ay / length,
|
||||
w: box.w * length,
|
||||
h: box.h * Math.hypot(bx, by),
|
||||
score: box.score,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function recognize(image, boxes) {
|
||||
const frames = boxes.map((box) => {
|
||||
const frame = readingFrame(box);
|
||||
return { frame, ratio: frame.width / Math.max(1, frame.height) };
|
||||
});
|
||||
const order = frames
|
||||
.map((_, i) => i)
|
||||
.sort((a, b) => frames[a].ratio - frames[b].ratio);
|
||||
const texts = new Array(frames.length);
|
||||
for (let start = 0; start < order.length; start += REC_BATCH) {
|
||||
const batch = order.slice(start, start + REC_BATCH);
|
||||
const maxRatio = Math.max(
|
||||
320 / REC_HEIGHT,
|
||||
...batch.map((i) => frames[i].ratio),
|
||||
);
|
||||
const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio));
|
||||
const plane = REC_HEIGHT * batchWidth;
|
||||
const input = new Float32Array(batch.length * 3 * plane);
|
||||
batch.forEach((i, n) => {
|
||||
const cropWidth = Math.min(
|
||||
batchWidth,
|
||||
Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)),
|
||||
);
|
||||
sampleCrop(
|
||||
image,
|
||||
frames[i].frame,
|
||||
input,
|
||||
n * 3 * plane,
|
||||
batchWidth,
|
||||
cropWidth,
|
||||
);
|
||||
});
|
||||
const result = await rec.run({
|
||||
[rec.inputNames[0]]: new ort.Tensor("float32", input, [
|
||||
batch.length,
|
||||
3,
|
||||
REC_HEIGHT,
|
||||
batchWidth,
|
||||
]),
|
||||
});
|
||||
const output = result[rec.outputNames[0]];
|
||||
ctcDecode(output.data, output.dims, chars).forEach((value, n) => {
|
||||
texts[batch[n]] = value;
|
||||
});
|
||||
}
|
||||
return boxes.map((box, i) => {
|
||||
const { frame } = frames[i];
|
||||
const xs = [
|
||||
frame.x,
|
||||
frame.x + frame.dx * frame.width,
|
||||
frame.x + frame.ex * frame.height,
|
||||
frame.x + frame.dx * frame.width + frame.ex * frame.height,
|
||||
];
|
||||
const ys = [
|
||||
frame.y,
|
||||
frame.y + frame.dy * frame.width,
|
||||
frame.y + frame.ey * frame.height,
|
||||
frame.y + frame.dy * frame.width + frame.ey * frame.height,
|
||||
];
|
||||
return {
|
||||
text: texts[i].text,
|
||||
score: texts[i].score,
|
||||
angle: Math.atan2(frame.dy, frame.dx),
|
||||
cx: (Math.min(...xs) + Math.max(...xs)) / 2,
|
||||
cy: (Math.min(...ys) + Math.max(...ys)) / 2,
|
||||
x0: Math.min(...xs),
|
||||
y0: Math.min(...ys),
|
||||
x1: Math.max(...xs),
|
||||
y1: Math.max(...ys),
|
||||
height: frame.height,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function decode(buffer, rotate = 0) {
|
||||
const { data, info } = await sharp(buffer, { failOn: "none" })
|
||||
.rotate() // honour EXIF orientation
|
||||
.rotate(rotate)
|
||||
.resize(2400, 2400, { fit: "inside", withoutEnlargement: true })
|
||||
.removeAlpha()
|
||||
.raw()
|
||||
.toBuffer({ resolveWithObject: true });
|
||||
return { data, width: info.width, height: info.height };
|
||||
}
|
||||
|
||||
async function readImage(image) {
|
||||
const boxes = await detect(image);
|
||||
const lines = (await recognize(image, boxes)).filter((line) =>
|
||||
line.text.trim(),
|
||||
);
|
||||
const score = lines.reduce(
|
||||
(sum, line) => sum + line.score * line.text.length,
|
||||
0,
|
||||
);
|
||||
return { lines, score, boxes };
|
||||
}
|
||||
|
||||
/** Returns recognised lines with their bounding boxes, in image pixels. */
|
||||
async function read(buffer) {
|
||||
const image = await decode(buffer);
|
||||
let best = await readImage(image);
|
||||
// A label photographed sideways yields mostly tall, narrow boxes; try
|
||||
// both quarter turns and keep whichever reads more confidently.
|
||||
const tall = best.boxes.filter((box) => {
|
||||
const frame = readingFrame(box);
|
||||
return frame.height > frame.width * 1.5;
|
||||
}).length;
|
||||
if (best.boxes.length && tall > best.boxes.length / 2)
|
||||
for (const angle of [90, 270]) {
|
||||
const attempt = await readImage(await decode(buffer, angle));
|
||||
if (attempt.score > best.score) best = attempt;
|
||||
}
|
||||
return best.lines;
|
||||
}
|
||||
|
||||
return { read };
|
||||
}
|
||||
461
server/ocr/label.js
Normal file
461
server/ocr/label.js
Normal file
|
|
@ -0,0 +1,461 @@
|
|||
// Turns OCR text boxes from a Nutrition Facts panel into a food. Built for
|
||||
// US/Canadian labels, with the common EU "per 100 g" table as a fallback.
|
||||
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
|
||||
|
||||
// FDA Daily Values (2016 rule) in the unit printed on labels; a %DV next to
|
||||
// an amount is used to catch misread digits and to fill amounts that are only
|
||||
// given as a percentage.
|
||||
const dailyValues = {
|
||||
fat: [78, "g"],
|
||||
"saturated-fat": [20, "g"],
|
||||
cholesterol: [300, "mg"],
|
||||
sodium: [2300, "mg"],
|
||||
carbohydrates: [275, "g"],
|
||||
fiber: [28, "g"],
|
||||
"added-sugars": [50, "g"],
|
||||
proteins: [50, "g"],
|
||||
"vitamin-d": [20, "mcg"],
|
||||
calcium: [1300, "mg"],
|
||||
iron: [18, "mg"],
|
||||
potassium: [4700, "mg"],
|
||||
"vitamin-a": [900, "mcg"],
|
||||
"vitamin-c": [90, "mg"],
|
||||
"vitamin-e": [15, "mg"],
|
||||
"vitamin-k": [120, "mcg"],
|
||||
"vitamin-b1": [1.2, "mg"],
|
||||
"vitamin-b2": [1.3, "mg"],
|
||||
"vitamin-pp": [16, "mg"],
|
||||
"vitamin-b6": [1.7, "mg"],
|
||||
"vitamin-b9": [400, "mcg"],
|
||||
"vitamin-b12": [2.4, "mcg"],
|
||||
biotin: [30, "mcg"],
|
||||
"pantothenic-acid": [5, "mg"],
|
||||
phosphorus: [1250, "mg"],
|
||||
iodine: [150, "mcg"],
|
||||
magnesium: [420, "mg"],
|
||||
zinc: [11, "mg"],
|
||||
selenium: [55, "mcg"],
|
||||
copper: [0.9, "mg"],
|
||||
manganese: [2.3, "mg"],
|
||||
chromium: [35, "mcg"],
|
||||
molybdenum: [45, "mcg"],
|
||||
chloride: [2300, "mg"],
|
||||
choline: [550, "mg"],
|
||||
};
|
||||
|
||||
// Most specific first: "saturated fat" must claim its row before "fat" does.
|
||||
const fields = [
|
||||
["trans-fat", /\btrans\.?\s*fat|\bgrasas?\s*trans\b/],
|
||||
[
|
||||
"saturated-fat",
|
||||
/\b(?:of which\s*)?sat(?:urated|\.)?\s*fat|\bsaturates\b|\bgrasas?\s*saturadas?\b/,
|
||||
],
|
||||
["polyunsaturated-fat", /\bpoly(?:unsat(?:urated|\.)?)?\s*fat/],
|
||||
["monounsaturated-fat", /\bmono(?:unsat(?:urated|\.)?)?\s*fat/],
|
||||
[
|
||||
"fat",
|
||||
/\b(?:total\s*)?fat\b(?!\s*free)|\bgrasas?(?:\s*totales?)?(?:\s*\(?lipidos\)?)?/,
|
||||
],
|
||||
["cholesterol", /\bcholest(?:erol|\.)?|\bcolesterol\b/],
|
||||
["sodium", /\bsodium\b|\bsodio\b/],
|
||||
["salt", /\bsalt\b/],
|
||||
["soluble-fiber", /(?<!in)\bsoluble\s*fib(?:er|re)/],
|
||||
["insoluble-fiber", /\binsoluble\s*fib(?:er|re)/],
|
||||
["fiber", /\b(?:dietary\s*)?fib(?:er|re)\b|\bfibra(?:\s*dietetica)?\b/],
|
||||
["polyols", /\bsugar\s*alcohols?\b|\bpolyols\b/],
|
||||
[
|
||||
"sugars",
|
||||
/\b(?:total\s*|of which\s*)?sugars?\b|\bazucares(?:\s*totales)?\b(?!\s*anadidos)/,
|
||||
],
|
||||
[
|
||||
"carbohydrates",
|
||||
/\b(?:total\s*)?carb(?:ohydrates?|s|\.)?|\bcarbohidratos(?:\s*\(?hidratos\s*(?:\d\s*)?de\s*carbono\)?)?|\bhidratos\s*de\s*carbono\b/,
|
||||
],
|
||||
["proteins", /\bprotein\b|\bproteinas?\b/],
|
||||
["vitamin-d", /\bvit(?:amin|amina|\.)?\s*d\d?\b/],
|
||||
["vitamin-a", /\bvit(?:amin|\.)?\s*a\b/],
|
||||
["vitamin-c", /\bvit(?:amin|\.)?\s*c\b/],
|
||||
["vitamin-e", /\bvit(?:amin|\.)?\s*e\b/],
|
||||
["vitamin-k", /\bvit(?:amin|\.)?\s*k\d?\b/],
|
||||
["vitamin-b6", /\bvit(?:amin|\.)?\s*b\s*-?6\b/],
|
||||
["vitamin-b12", /\bvit(?:amin|\.)?\s*b\s*-?12\b/],
|
||||
["vitamin-b1", /\bthiamine?\b/],
|
||||
["vitamin-b2", /\briboflavin\b/],
|
||||
["vitamin-pp", /\bniacin\b/],
|
||||
["vitamin-b9", /\bfolate\b|\bfolic acid\b/],
|
||||
["biotin", /\bbiotin\b/],
|
||||
["pantothenic-acid", /\bpantothenic\b/],
|
||||
["calcium", /\bcalcium\b|\bcalcio\b/],
|
||||
["iron", /\biron\b|\bhierro\b/],
|
||||
["potassium", /\bpotas(?:sium|\.)?|\bpotasio\b/],
|
||||
["phosphorus", /\bphosphorus\b/],
|
||||
["magnesium", /\bmagnesium\b/],
|
||||
["zinc", /\bzinc\b/],
|
||||
["selenium", /\bselenium\b/],
|
||||
["copper", /\bcopper\b/],
|
||||
["manganese", /\bmanganese\b/],
|
||||
["iodine", /\biodine\b/],
|
||||
["chromium", /\bchromium\b/],
|
||||
["molybdenum", /\bmolybdenum\b/],
|
||||
["chloride", /\bchloride\b/],
|
||||
["choline", /\bcholine\b/],
|
||||
["caffeine", /\bcaffeine\b/],
|
||||
];
|
||||
// The unit printed when OCR loses it.
|
||||
const defaultUnits = {
|
||||
"energy-kcal": "kcal",
|
||||
sodium: "mg",
|
||||
cholesterol: "mg",
|
||||
potassium: "mg",
|
||||
calcium: "mg",
|
||||
iron: "mg",
|
||||
"vitamin-d": "mcg",
|
||||
};
|
||||
|
||||
/**
|
||||
* Groups boxes into reading rows. Coordinates are first turned by the
|
||||
* dominant text angle, so a tilted photo still lines up "Sodium 160mg 7%".
|
||||
*/
|
||||
export function rows(lines) {
|
||||
const angles = lines
|
||||
.filter(
|
||||
(line) => line.angle !== undefined && line.x1 - line.x0 > 3 * line.height,
|
||||
)
|
||||
.map((line) => line.angle)
|
||||
.sort((a, b) => a - b);
|
||||
const angle = angles.length ? angles[Math.floor(angles.length / 2)] : 0;
|
||||
const cos = Math.cos(angle);
|
||||
const sin = Math.sin(angle);
|
||||
const placed = lines.map((line) => {
|
||||
const cx = line.cx ?? (line.x0 + line.x1) / 2;
|
||||
const cy = line.cy ?? (line.y0 + line.y1) / 2;
|
||||
const height = line.height ?? line.y1 - line.y0;
|
||||
const y = -cx * sin + cy * cos;
|
||||
return {
|
||||
text: line.text,
|
||||
x: cx * cos + cy * sin,
|
||||
y0: y - height / 2,
|
||||
y1: y + height / 2,
|
||||
};
|
||||
});
|
||||
placed.sort((a, b) => a.y0 + a.y1 - b.y0 - b.y1);
|
||||
const result = [];
|
||||
for (const line of placed) {
|
||||
const height = line.y1 - line.y0;
|
||||
const row = result.find((candidate) => {
|
||||
const overlap =
|
||||
Math.min(candidate.y1, line.y1) - Math.max(candidate.y0, line.y0);
|
||||
return overlap > 0.5 * Math.min(height, candidate.y1 - candidate.y0);
|
||||
});
|
||||
if (row) row.items.push(line);
|
||||
else result.push({ y0: line.y0, y1: line.y1, items: [line] });
|
||||
}
|
||||
return result
|
||||
.sort((a, b) => a.y0 - b.y0)
|
||||
.map((row) => {
|
||||
row.items.sort((a, b) => a.x - b.x);
|
||||
return clean(row.items.map((item) => item.text).join(" "));
|
||||
});
|
||||
}
|
||||
|
||||
// Label vocabulary; OCR misspellings within a small edit distance of these
|
||||
// ("Sodlum", "Proteln", "Caicium") are snapped back before matching.
|
||||
const vocabulary = `calories total fat saturated trans polyunsaturated
|
||||
monounsaturated cholesterol sodium carbohydrate carbohydrates dietary fiber
|
||||
soluble insoluble sugars sugar added includes alcohols protein vitamin
|
||||
calcium iron potassium serving servings size container amount daily value
|
||||
thiamin riboflavin niacin folate phosphorus magnesium zinc selenium copper
|
||||
manganese biotin pantothenic iodine chromium molybdenum chloride choline
|
||||
caffeine energy saturates fibre salt nutrition facts grasa grasas saturada
|
||||
colesterol sodio fibra dietetica azucares anadidos carbohidratos hidratos
|
||||
carbono proteinas calcio hierro potasio vitamina energetico contenido
|
||||
porcion lipidos`.split(/\s+/);
|
||||
const known = new Set(vocabulary);
|
||||
function distance(a, b, limit) {
|
||||
if (Math.abs(a.length - b.length) > limit) return limit + 1;
|
||||
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
const current = [i];
|
||||
let best = i;
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
current[j] = Math.min(
|
||||
previous[j] + 1,
|
||||
current[j - 1] + 1,
|
||||
previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
|
||||
);
|
||||
best = Math.min(best, current[j]);
|
||||
}
|
||||
if (best > limit) return limit + 1;
|
||||
previous = current;
|
||||
}
|
||||
return previous[b.length];
|
||||
}
|
||||
function correctWord(word) {
|
||||
const lower = word.toLowerCase();
|
||||
if (known.has(lower)) return word;
|
||||
const limit = lower.length >= 8 ? 2 : 1;
|
||||
let best = null;
|
||||
let bestDistance = limit + 1;
|
||||
for (const candidate of vocabulary) {
|
||||
const d = distance(lower, candidate, limit);
|
||||
if (d < bestDistance) {
|
||||
best = candidate;
|
||||
bestDistance = d;
|
||||
}
|
||||
}
|
||||
return best ?? word;
|
||||
}
|
||||
|
||||
/** Undoes the usual OCR confusions in label text. */
|
||||
export function clean(text) {
|
||||
return (
|
||||
text
|
||||
.normalize("NFD")
|
||||
.replace(/[̀-ͯ]/g, "") // Azúcares → Azucares
|
||||
.replace(/[a-z]{5,}/gi, correctWord)
|
||||
.replace(/[µμ]g\b/gi, "mcg")
|
||||
.replace(/\bug\b/gi, "mcg")
|
||||
.replace(/less than/gi, "<")
|
||||
// Letters standing in for digits next to digits or units: "Og", "1O".
|
||||
.replace(/(?<=\d)[oO]|[oO](?=\d)/g, "0")
|
||||
.replace(/(?<![a-z])[oO](?=\s?(?:m?c?g|%)\b)/g, "0")
|
||||
.replace(/(?<=\d)[lI|](?=\d|\s?m?g\b)|(?<![a-z])[lI|](?=\d)/g, "1")
|
||||
// A footnote mark read as a digit: "Carbohydrate1 13g".
|
||||
.replace(/\b([a-z]{4,})[1†*]\s+(?=<?\d)/gi, "$1 ")
|
||||
// Thousands separators, then decimal commas.
|
||||
.replace(/(\d),(\d{3})(?!\d)/g, "$1$2")
|
||||
.replace(/(\d),(\d)/g, "$1.$2")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
);
|
||||
}
|
||||
|
||||
// The amount may follow a split word ("Carbohy ydrate 42g"); a number
|
||||
// directly followed by % is a percentage, not an amount.
|
||||
const amountPattern =
|
||||
/^\s*((?:[a-z]{1,10}\s+){0,2})[\s:.\-–—]*(<\s*)?(\d+(?:\.\d+)?|\.\d+)\s*(mcg|mg|g|kcal|kj|iu)?(?![a-z%\d.])(?!\s*%)/i;
|
||||
const percentPattern = /^[^\d%]{0,12}?(\d{1,4})\s*%/;
|
||||
|
||||
function readAmount(rest) {
|
||||
const found = amountPattern.exec(rest);
|
||||
if (!found) return null;
|
||||
// Never skip over another nutrient's name: in "Cholest. Protein 9g" the
|
||||
// 9g is protein's.
|
||||
const skipped = found[1].trim().toLowerCase().split(/\s+/).filter(Boolean);
|
||||
if (skipped.some((word) => known.has(word))) return null;
|
||||
const match = [found[0], ...found.slice(2)];
|
||||
const after = rest.slice(match[0].length);
|
||||
const percent = percentPattern.exec(after);
|
||||
return {
|
||||
less: Boolean(match[1]),
|
||||
raw: match[2],
|
||||
value: Number(match[2]),
|
||||
unit: match[3]?.toLowerCase(),
|
||||
percent: percent ? Number(percent[1]) : null,
|
||||
};
|
||||
}
|
||||
|
||||
const toCatalog = (key, value, unit) =>
|
||||
key === "energy-kcal"
|
||||
? value
|
||||
: convertMass(value, unit === "mcg" ? "µg" : unit, nutrientUnit(key));
|
||||
|
||||
/** Resolves an amount, using its %DV to correct misreads such as 8g → "89". */
|
||||
function resolve(key, reading) {
|
||||
const dv = dailyValues[key];
|
||||
let unit = reading.unit || defaultUnits[key] || dv?.[1] || "g";
|
||||
if (unit === "iu" || unit === "kj" || unit === "kcal") return null;
|
||||
if (unit === "g" && dv?.[1] === "mcg") unit = "mcg"; // "mcg" read as "g"
|
||||
let value = reading.less ? reading.value / 2 : reading.value;
|
||||
if (dv && reading.percent !== null) {
|
||||
const daily = convertMass(dv[0], mass(dv[1]), mass(unit));
|
||||
// %DV is rounded (and may read 0 below 2%), so compare in points.
|
||||
const consistent = (candidate) =>
|
||||
Math.abs((candidate / daily) * 100 - reading.percent) <=
|
||||
Math.max(2, reading.percent * 0.3);
|
||||
if (daily && !consistent(value)) {
|
||||
const candidates = [];
|
||||
// A unit read as a digit: "8g" → "89", "160mg" → "16019".
|
||||
if (!reading.unit && /\d9$/.test(reading.raw))
|
||||
candidates.push(Number(reading.raw.slice(0, -1)));
|
||||
if (!reading.unit && /\d19$/.test(reading.raw))
|
||||
candidates.push(Number(reading.raw.slice(0, -2)));
|
||||
// A lost decimal point: "05" → 0.5, "25" → 2.5.
|
||||
if (/^\d{2,3}$/.test(reading.raw))
|
||||
candidates.push(Number(reading.raw) / 10);
|
||||
const fixed = candidates.find(consistent);
|
||||
if (fixed !== undefined) value = fixed;
|
||||
// Nothing plausible was read; trust the percentage.
|
||||
else if (!reading.unit) value = (reading.percent / 100) * daily;
|
||||
}
|
||||
}
|
||||
const result = toCatalog(key, value, unit);
|
||||
// Far beyond a day's worth in one serving is a misread, not a food.
|
||||
if (dv && result > 10 * toCatalog(key, dv[0], dv[1])) return null;
|
||||
return result;
|
||||
}
|
||||
const mass = (unit) => (unit === "mcg" ? "µg" : unit);
|
||||
|
||||
function percentOnly(key, rest) {
|
||||
const dv = dailyValues[key];
|
||||
const match = /^[^\d]{0,6}(\d{1,4})\s*%/.exec(rest);
|
||||
if (!dv || !match) return null;
|
||||
return toCatalog(key, (Number(match[1]) / 100) * dv[0], dv[1]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{text: string, x0: number, y0: number, x1: number, y1: number}[]} lines
|
||||
* @returns {{serving?: string, nutrients: Record<string, number>,
|
||||
* extra: Record<string, number>, found: string[], text: string[]}}
|
||||
*/
|
||||
export function parseLabel(lines) {
|
||||
const text = rows(lines);
|
||||
const values = {};
|
||||
const claimed = new Set(); // character spans already used, per row
|
||||
const used = (row, start, end) => {
|
||||
for (let i = start; i < end; i++)
|
||||
if (claimed.has(`${row}:${i}`)) return true;
|
||||
for (let i = start; i < end; i++) claimed.add(`${row}:${i}`);
|
||||
return false;
|
||||
};
|
||||
|
||||
// "Includes 10g Added Sugars" puts the amount before the name.
|
||||
text.forEach((row, index) => {
|
||||
const match =
|
||||
/\bincl(?:udes|\.)?\s*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?\s*(?:of\s*)?added\s*sugars?/i.exec(
|
||||
row,
|
||||
) ||
|
||||
/\b(?:added\s*sugars?|azucares\s*anadidos)[\s:]*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?/i.exec(
|
||||
row,
|
||||
);
|
||||
if (!match || values["added-sugars"] !== undefined) return;
|
||||
used(index, match.index, match.index + match[0].length);
|
||||
const percent = percentPattern.exec(
|
||||
row.slice(match.index + match[0].length),
|
||||
);
|
||||
values["added-sugars"] = resolve("added-sugars", {
|
||||
less: Boolean(match[1]),
|
||||
raw: match[2],
|
||||
value: Number(match[2]),
|
||||
unit: match[3],
|
||||
percent: percent ? Number(percent[1]) : null,
|
||||
});
|
||||
});
|
||||
|
||||
for (const [key, pattern] of fields) {
|
||||
if (values[key] !== undefined) continue;
|
||||
for (
|
||||
let index = 0;
|
||||
index < text.length && values[key] === undefined;
|
||||
index++
|
||||
) {
|
||||
const regex = new RegExp(pattern.source, "gi");
|
||||
for (let match; (match = regex.exec(text[index])); ) {
|
||||
const start = match.index;
|
||||
const end = start + match[0].length;
|
||||
if (used(index, start, end)) continue;
|
||||
let rest = text[index].slice(end);
|
||||
// A linear label can wrap between a name and its amount.
|
||||
if (
|
||||
/^[\s,.:]*$/.test(rest) &&
|
||||
/^<?\s*\.?\d/.test(text[index + 1] || "")
|
||||
)
|
||||
rest = " " + text[index + 1];
|
||||
const reading = readAmount(rest);
|
||||
const value = reading ? resolve(key, reading) : percentOnly(key, rest);
|
||||
if (value === null || value === undefined) continue;
|
||||
values[key] = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (values.salt !== undefined) {
|
||||
if (values.sodium === undefined) values.sodium = (values.salt / 2.5) * 1000;
|
||||
delete values.salt;
|
||||
}
|
||||
|
||||
// Calories: the amount follows on the same row, or sits alone on a
|
||||
// neighbouring row when it's printed in a much larger font.
|
||||
for (let index = 0; index < text.length; index++) {
|
||||
const row = text[index];
|
||||
const match =
|
||||
/\bcalories\b(?!\s*from)|\benergy\b|\benerg(?:ia|etico)\b/i.exec(row);
|
||||
if (!match) continue;
|
||||
const rest = row.slice(match.index + match[0].length);
|
||||
const kcal =
|
||||
/(\d+(?:\.\d+)?)\s*kcal/i.exec(rest) ||
|
||||
/^[^\d]{0,24}?(\d{1,4})(?!\s*(?:k?j|%|m?c?g))\b/i.exec(rest);
|
||||
const before = text[index - 1] || "";
|
||||
const after = text[index + 1] || "";
|
||||
const perServing =
|
||||
/amount\s*per\s*serving[:\s]*(\d{1,4})\b(?!\s*(?:m?c?g|%))/i;
|
||||
const alone =
|
||||
// Big type often lands beside "Amount per serving" on the next row.
|
||||
perServing.exec(before) ||
|
||||
perServing.exec(after) ||
|
||||
// A number alone at the end of a neighbouring row.
|
||||
[after, before]
|
||||
.map((neighbour) =>
|
||||
/%|\d\s*m?c?g\b/i.test(neighbour)
|
||||
? null
|
||||
: /(?:^|\s)(\d{1,4})$/.exec(neighbour),
|
||||
)
|
||||
.find(Boolean) ||
|
||||
// A linear label wrapped right after "Calories": "80, Total Fat…".
|
||||
/^(\d{1,4})\b(?!\s*(?:m?c?g|%|\.\d))/.exec(after);
|
||||
const value = Number((kcal || alone)?.[1]);
|
||||
if (Number.isFinite(value) && value <= 5000 && (kcal || alone)) {
|
||||
values["energy-kcal"] = value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let serving;
|
||||
const perHundred = text.some((row) => /\bper\s*100\s*(g|ml)\b/i.test(row));
|
||||
for (let index = 0; index < text.length; index++) {
|
||||
const match = /\bserv(?:ing|\.)\s*size\b[\s:]*(.*)$/i.exec(text[index]);
|
||||
if (!match) continue;
|
||||
let size = match[1].trim();
|
||||
if (!/\d/.test(size)) size = `${size} ${text[index + 1] || ""}`.trim();
|
||||
// Two-column layouts can run nutrient rows into the serving text.
|
||||
size = size
|
||||
.split(/\b(?:calories|amount|total|sodium|protein|servings)\b|%/i)[0]
|
||||
.trim()
|
||||
.replace(/\s*\(\s*/g, " (")
|
||||
.replace(/\s*\)\s*/g, ")")
|
||||
.replace(/(\d)\s*(g|ml)\b/gi, "$1 $2")
|
||||
.replace(/\)\s*.*$/, ")")
|
||||
.trim();
|
||||
if (size && /\d/.test(size)) serving = size.slice(0, 100);
|
||||
break;
|
||||
}
|
||||
if (!serving && perHundred) {
|
||||
const ml = text.some((row) => /\bper\s*100\s*ml\b/i.test(row));
|
||||
serving = ml ? "100 ml" : "100 g";
|
||||
}
|
||||
|
||||
const core = {
|
||||
"energy-kcal": "calories",
|
||||
proteins: "protein",
|
||||
carbohydrates: "carbs",
|
||||
fat: "fat",
|
||||
fiber: "fiber",
|
||||
sugars: "sugar",
|
||||
sodium: "sodium",
|
||||
};
|
||||
const nutrients = {};
|
||||
const extra = {};
|
||||
for (const [key, value] of Object.entries(values)) {
|
||||
if (!Number.isFinite(value) || value < 0 || value > 100000) continue;
|
||||
const rounded = Math.round(value * 1000) / 1000;
|
||||
if (core[key]) nutrients[core[key]] = rounded;
|
||||
else extra[key] = rounded;
|
||||
}
|
||||
return {
|
||||
...(serving ? { serving } : {}),
|
||||
nutrients,
|
||||
extra,
|
||||
found: Object.keys(values),
|
||||
text,
|
||||
};
|
||||
}
|
||||
165
server/products/db.js
Normal file
165
server/products/db.js
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { titleCase } from "../../shared/products.js";
|
||||
|
||||
// Each source lives in its own table so a refresh can be built alongside the
|
||||
// old copy and swapped in atomically. The file is a cache: it can be deleted
|
||||
// at any time and is rebuilt on the next import.
|
||||
export const sources = ["usda", "off", "api"];
|
||||
const columns = `gtin TEXT PRIMARY KEY, name TEXT, brand TEXT, serving TEXT,
|
||||
serving_qty REAL, serving_unit TEXT, nutrients TEXT`;
|
||||
|
||||
export function openProducts(filename) {
|
||||
const db = new DatabaseSync(filename);
|
||||
db.exec(`PRAGMA journal_mode=WAL; PRAGMA busy_timeout=15000;
|
||||
CREATE TABLE IF NOT EXISTS imports (source TEXT PRIMARY KEY, version TEXT,
|
||||
imported_at TEXT, rows INTEGER, checked_at TEXT, error TEXT);`);
|
||||
for (const source of ["usda", "off"])
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${source}_products (${columns}) WITHOUT ROWID`,
|
||||
);
|
||||
// API answers include misses (nutrients NULL) so repeated scans of an
|
||||
// unknown code don't hit Open Food Facts again.
|
||||
db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS api_products (${columns}, fetched_at INTEGER NOT NULL) WITHOUT ROWID`,
|
||||
);
|
||||
const select = Object.fromEntries(
|
||||
sources.map((source) => [
|
||||
source,
|
||||
db.prepare(`SELECT * FROM ${source}_products WHERE gtin=?`),
|
||||
]),
|
||||
);
|
||||
const toProduct = (row, source) =>
|
||||
row?.nutrients
|
||||
? {
|
||||
gtin: row.gtin,
|
||||
name: source === "usda" ? titleCase(row.name) : row.name,
|
||||
brand: source === "usda" ? titleCase(row.brand) : row.brand,
|
||||
serving: row.serving,
|
||||
servingQty: row.serving_qty,
|
||||
servingUnit: row.serving_unit,
|
||||
nutrients: JSON.parse(row.nutrients),
|
||||
source,
|
||||
}
|
||||
: null;
|
||||
|
||||
/** The best local match, or `undefined` when no source has an answer. */
|
||||
function lookup(
|
||||
gtin,
|
||||
{ apiMaxAge = 30 * 86400000, missMaxAge = 86400000 } = {},
|
||||
) {
|
||||
const found = {};
|
||||
for (const source of sources) found[source] = select[source].get(gtin);
|
||||
const api = found.api;
|
||||
if (
|
||||
api &&
|
||||
Date.now() - api.fetched_at > (api.nutrients ? apiMaxAge : missMaxAge)
|
||||
)
|
||||
found.api = undefined;
|
||||
const usda = toProduct(found.usda, "usda");
|
||||
const off = toProduct(found.off, "off") || toProduct(found.api, "api");
|
||||
// USDA comes from manufacturers' own label submissions, so it wins;
|
||||
// community-entered Open Food Facts fills in what it lacks.
|
||||
const base = usda || off;
|
||||
if (base)
|
||||
return {
|
||||
...base,
|
||||
name: base.name || off?.name,
|
||||
brand: base.brand || off?.brand,
|
||||
};
|
||||
return found.api ? null : undefined;
|
||||
}
|
||||
|
||||
const upsertApi = db.prepare(
|
||||
`INSERT OR REPLACE INTO api_products VALUES (?,?,?,?,?,?,?,?)`,
|
||||
);
|
||||
function remember(gtin, product) {
|
||||
upsertApi.run(
|
||||
gtin,
|
||||
product?.name ?? null,
|
||||
product?.brand ?? null,
|
||||
product?.serving ?? null,
|
||||
product?.servingQty ?? null,
|
||||
product?.servingUnit ?? null,
|
||||
product ? JSON.stringify(product.nutrients) : null,
|
||||
Date.now(),
|
||||
);
|
||||
}
|
||||
|
||||
/** Bulk loader used by the importers: writes `<source>_products_next`. */
|
||||
function beginImport(source) {
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next;
|
||||
CREATE TABLE ${source}_products_next (${columns}) WITHOUT ROWID;`);
|
||||
const insert = db.prepare(
|
||||
`INSERT OR REPLACE INTO ${source}_products_next VALUES (?,?,?,?,?,?,?)`,
|
||||
);
|
||||
let pending = 0;
|
||||
let rows = 0;
|
||||
db.exec("BEGIN");
|
||||
return {
|
||||
add(product) {
|
||||
insert.run(
|
||||
product.gtin,
|
||||
product.name || null,
|
||||
product.brand || null,
|
||||
product.serving || null,
|
||||
product.servingQty ?? null,
|
||||
product.servingUnit || null,
|
||||
JSON.stringify(product.nutrients),
|
||||
);
|
||||
rows++;
|
||||
if (++pending >= 20000) {
|
||||
db.exec("COMMIT; BEGIN");
|
||||
pending = 0;
|
||||
}
|
||||
},
|
||||
commit(version) {
|
||||
db.exec(`COMMIT; BEGIN IMMEDIATE;
|
||||
DROP TABLE IF EXISTS ${source}_products;
|
||||
ALTER TABLE ${source}_products_next RENAME TO ${source}_products;`);
|
||||
db.prepare(
|
||||
`INSERT OR REPLACE INTO imports(source,version,imported_at,rows,checked_at,error)
|
||||
VALUES (?,?,?,?,?,NULL)`,
|
||||
).run(
|
||||
source,
|
||||
version,
|
||||
new Date().toISOString(),
|
||||
rows,
|
||||
new Date().toISOString(),
|
||||
);
|
||||
db.exec("COMMIT");
|
||||
// Old pages are reused by the next import; reclaiming them would
|
||||
// rewrite the whole file.
|
||||
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
||||
return rows;
|
||||
},
|
||||
abort() {
|
||||
if (db.isTransaction) db.exec("ROLLBACK");
|
||||
db.exec(`DROP TABLE IF EXISTS ${source}_products_next`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const state = (source) =>
|
||||
db.prepare("SELECT * FROM imports WHERE source=?").get(source);
|
||||
function recordCheck(source, error = null) {
|
||||
db.prepare(
|
||||
`INSERT INTO imports(source,checked_at,error) VALUES (?,?,?)
|
||||
ON CONFLICT(source) DO UPDATE SET checked_at=excluded.checked_at, error=excluded.error`,
|
||||
).run(source, new Date().toISOString(), error);
|
||||
}
|
||||
const status = () =>
|
||||
db
|
||||
.prepare("SELECT source, version, imported_at, rows, error FROM imports")
|
||||
.all();
|
||||
|
||||
return {
|
||||
db,
|
||||
lookup,
|
||||
remember,
|
||||
beginImport,
|
||||
state,
|
||||
recordCheck,
|
||||
status,
|
||||
close: () => db.close(),
|
||||
};
|
||||
}
|
||||
94
server/products/import.js
Normal file
94
server/products/import.js
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
// Refreshes the local product databases. Runs as a low-priority child process
|
||||
// (see scheduler.js) so parsing several gigabytes never stalls the app.
|
||||
// node server/products/import.js <data dir> [--force]
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { setPriority } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { openProducts } from "./db.js";
|
||||
import { OFF_CSV, importOffCsv, userAgent } from "./off.js";
|
||||
import { download, importUsdaZip, latestUsdaArchive } from "./usda.js";
|
||||
|
||||
const dataDir = resolve(process.argv[2] || "data");
|
||||
const force = process.argv.includes("--force");
|
||||
const OFF_MAX_AGE = 7 * 86400000;
|
||||
const log = (...args) => console.log("[products]", ...args);
|
||||
try {
|
||||
setPriority(19);
|
||||
} catch {
|
||||
// Lowering priority is best effort.
|
||||
}
|
||||
const products = openProducts(join(dataDir, "products.sqlite"));
|
||||
const tmp = join(dataDir, "tmp");
|
||||
mkdirSync(tmp, { recursive: true });
|
||||
|
||||
async function refreshUsda() {
|
||||
const latest = await latestUsdaArchive();
|
||||
if (!force && products.state("usda")?.version === latest.version)
|
||||
return products.recordCheck("usda");
|
||||
const file = join(tmp, "usda-branded.zip");
|
||||
log(`downloading USDA Branded Foods ${latest.version}`);
|
||||
try {
|
||||
await download(latest.url, file);
|
||||
const sink = products.beginImport("usda");
|
||||
try {
|
||||
await importUsdaZip(file, sink, {
|
||||
onProgress: (n) => log(`USDA: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`USDA: imported ${sink.commit(latest.version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
} finally {
|
||||
rmSync(file, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshOff() {
|
||||
const state = products.state("off");
|
||||
if (
|
||||
!force &&
|
||||
state?.imported_at &&
|
||||
Date.now() - Date.parse(state.imported_at) < OFF_MAX_AGE
|
||||
)
|
||||
return;
|
||||
const response = await fetch(OFF_CSV, {
|
||||
headers: { "User-Agent": userAgent },
|
||||
});
|
||||
if (!response.ok)
|
||||
throw new Error(`Open Food Facts answered ${response.status}`);
|
||||
const version =
|
||||
response.headers.get("last-modified") || new Date().toUTCString();
|
||||
if (!force && state?.version === version) {
|
||||
await response.body?.cancel();
|
||||
return products.recordCheck("off");
|
||||
}
|
||||
log("streaming the Open Food Facts export");
|
||||
const sink = products.beginImport("off");
|
||||
try {
|
||||
await importOffCsv(response, sink, {
|
||||
onProgress: (n) => log(`Open Food Facts: ${n.toLocaleString()} products`),
|
||||
});
|
||||
log(
|
||||
`Open Food Facts: imported ${sink.commit(version).toLocaleString()} products`,
|
||||
);
|
||||
} catch (error) {
|
||||
sink.abort();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [source, refresh] of [
|
||||
["usda", refreshUsda],
|
||||
["off", refreshOff],
|
||||
]) {
|
||||
try {
|
||||
await refresh();
|
||||
} catch (error) {
|
||||
console.error(`[products] ${source} refresh failed:`, error.message);
|
||||
products.recordCheck(source, String(error.message).slice(0, 300));
|
||||
}
|
||||
}
|
||||
products.close();
|
||||
175
server/products/off.js
Normal file
175
server/products/off.js
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { createGunzip } from "node:zlib";
|
||||
import { Readable } from "node:stream";
|
||||
import { databaseGtin } from "../../shared/barcode.js";
|
||||
import { fromGrams, nutrientKeyPattern } from "../../shared/nutrients.js";
|
||||
|
||||
export const OFF_CSV =
|
||||
"https://static.openfoodfacts.org/data/en.openfoodfacts.org.products.csv.gz";
|
||||
export const userAgent =
|
||||
"Calio/0.1 (self-hosted personal nutrition journal; https://git.elijahkuntz.com/Elijah/Calio)";
|
||||
|
||||
// Columns that are not nutrient amounts, or duplicate one handled separately.
|
||||
const skipped = new Set([
|
||||
"energy",
|
||||
"energy-kj",
|
||||
"energy-from-fat",
|
||||
"salt",
|
||||
"added-salt",
|
||||
"ph",
|
||||
"alcohol", // % vol in Open Food Facts, not grams
|
||||
"cocoa",
|
||||
"chlorophyl",
|
||||
"collagen-meat-protein-ratio",
|
||||
"glycemic-index",
|
||||
"water-hardness",
|
||||
"acidity",
|
||||
"carbohydrates-total",
|
||||
]);
|
||||
const notNutrient = /score|estimate|footprint|fruits-vegetables|nova|prepared/;
|
||||
|
||||
/**
|
||||
* Converts Open Food Facts per-100 g values (grams, kcal) into Calio's
|
||||
* stored units.
|
||||
* @param {Iterable<[string, unknown]>} entries `[key, value]` without `_100g`
|
||||
*/
|
||||
export function offNutrients(entries) {
|
||||
const raw = new Map();
|
||||
for (const [key, value] of entries) {
|
||||
const number = typeof value === "number" ? value : Number(value);
|
||||
if (value === "" || value == null || !Number.isFinite(number) || number < 0)
|
||||
continue;
|
||||
raw.set(key, number);
|
||||
}
|
||||
const nutrients = {};
|
||||
let kcal = raw.get("energy-kcal");
|
||||
if (kcal === undefined) {
|
||||
const kj = raw.get("energy-kj") ?? raw.get("energy");
|
||||
if (kj !== undefined) kcal = kj / 4.184;
|
||||
}
|
||||
if (kcal !== undefined && kcal <= 950) nutrients["energy-kcal"] = kcal;
|
||||
let sodium = raw.get("sodium");
|
||||
if (sodium === undefined && raw.has("salt")) sodium = raw.get("salt") / 2.5;
|
||||
if (sodium !== undefined && sodium <= 100) nutrients.sodium = sodium * 1000;
|
||||
for (const [key, grams] of raw) {
|
||||
if (
|
||||
key === "energy-kcal" ||
|
||||
key === "sodium" ||
|
||||
skipped.has(key) ||
|
||||
notNutrient.test(key) ||
|
||||
!nutrientKeyPattern.test(key) ||
|
||||
grams > 100 // more than 100 g per 100 g is a data-entry error
|
||||
)
|
||||
continue;
|
||||
const value = fromGrams(key, grams);
|
||||
if (value !== null) nutrients[key] = value;
|
||||
}
|
||||
return nutrients;
|
||||
}
|
||||
|
||||
function servingUnit(text) {
|
||||
return /\d\s*(ml|millilit|fl\.?\s*oz|cl\b|l\b)/i.test(text || "")
|
||||
? "ml"
|
||||
: "g";
|
||||
}
|
||||
|
||||
/** Shapes a CSV row or API product into a products-table row. */
|
||||
export function offProduct({
|
||||
code,
|
||||
name,
|
||||
brands,
|
||||
serving,
|
||||
servingQty,
|
||||
nutrients,
|
||||
}) {
|
||||
const gtin = databaseGtin(code);
|
||||
if (!gtin || nutrients["energy-kcal"] === undefined) return null;
|
||||
const qty = Number(servingQty);
|
||||
return {
|
||||
gtin,
|
||||
name: (name || "").trim().slice(0, 200) || null,
|
||||
brand: (brands || "").split(",")[0].trim().slice(0, 100) || null,
|
||||
serving: (serving || "").trim().slice(0, 100) || null,
|
||||
servingQty: Number.isFinite(qty) && qty > 0 ? qty : null,
|
||||
servingUnit: servingUnit(serving),
|
||||
nutrients,
|
||||
};
|
||||
}
|
||||
|
||||
async function* lines(stream) {
|
||||
let rest = "";
|
||||
stream.setEncoding("utf8");
|
||||
for await (const chunk of stream) {
|
||||
const parts = (rest + chunk).split("\n");
|
||||
rest = parts.pop();
|
||||
yield* parts;
|
||||
}
|
||||
if (rest) yield rest;
|
||||
}
|
||||
|
||||
/** Streams the daily CSV export (~1.5 GB gzipped) straight into `sink`. */
|
||||
export async function importOffCsv(response, sink, { onProgress } = {}) {
|
||||
const gunzip = Readable.fromWeb(response.body).pipe(createGunzip());
|
||||
let header;
|
||||
let count = 0;
|
||||
for await (const line of lines(gunzip)) {
|
||||
const fields = line.split("\t");
|
||||
if (!header) {
|
||||
header = Object.fromEntries(fields.map((name, i) => [name, i]));
|
||||
for (const column of ["code", "product_name", "energy-kcal_100g"])
|
||||
if (!(column in header))
|
||||
throw new Error(`Open Food Facts export is missing ${column}`);
|
||||
header.nutrientColumns = fields
|
||||
.map((name, i) => [name, i])
|
||||
.filter(([name]) => name.endsWith("_100g"))
|
||||
.map(([name, i]) => [name.slice(0, -5), i]);
|
||||
continue;
|
||||
}
|
||||
if (fields[header.no_nutrition_data] === "on") continue;
|
||||
const product = offProduct({
|
||||
code: fields[header.code],
|
||||
name: fields[header.product_name],
|
||||
brands: fields[header.brands],
|
||||
serving: fields[header.serving_size],
|
||||
servingQty: fields[header.serving_quantity],
|
||||
nutrients: offNutrients(
|
||||
header.nutrientColumns.map(([key, i]) => [key, fields[i]]),
|
||||
),
|
||||
});
|
||||
if (!product) continue;
|
||||
sink.add(product);
|
||||
if (++count % 250000 === 0) onProgress?.(count);
|
||||
}
|
||||
if (!header) throw new Error("Open Food Facts export was empty");
|
||||
return count;
|
||||
}
|
||||
|
||||
const apiFields =
|
||||
"code,product_name,product_name_en,brands,serving_size,serving_quantity,nutriments";
|
||||
/** Live lookup, used only when no local copy has the product. */
|
||||
export async function fetchOffProduct(gtin, fetchImpl = fetch) {
|
||||
// Open Food Facts keys products by EAN-13 (or EAN-8).
|
||||
const stripped = gtin.replace(/^0+/, "");
|
||||
const code = stripped.padStart(stripped.length <= 8 ? 8 : 13, "0");
|
||||
const response = await fetchImpl(
|
||||
`https://world.openfoodfacts.org/api/v2/product/${code}.json?fields=${apiFields}`,
|
||||
{ headers: { "User-Agent": userAgent }, signal: AbortSignal.timeout(8000) },
|
||||
);
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok)
|
||||
throw new Error(`Open Food Facts answered ${response.status}`);
|
||||
const data = await response.json();
|
||||
if (data.status !== 1 || !data.product) return null;
|
||||
const p = data.product;
|
||||
const entries = Object.entries(p.nutriments || {})
|
||||
.filter(([key]) => key.endsWith("_100g"))
|
||||
.map(([key, value]) => [key.slice(0, -5), value]);
|
||||
const product = offProduct({
|
||||
code: gtin,
|
||||
name: p.product_name_en || p.product_name,
|
||||
brands: p.brands,
|
||||
serving: p.serving_size,
|
||||
servingQty: p.serving_quantity,
|
||||
nutrients: offNutrients(entries),
|
||||
});
|
||||
return product && { ...product, source: "api" };
|
||||
}
|
||||
27
server/products/scheduler.js
Normal file
27
server/products/scheduler.js
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import { fork } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const script = fileURLToPath(new URL("./import.js", import.meta.url));
|
||||
const HOUR = 3600000;
|
||||
|
||||
/**
|
||||
* Checks for new product data shortly after start-up and every six hours.
|
||||
* The import itself decides whether anything needs downloading.
|
||||
*/
|
||||
export function scheduleProductImports(dataDir, { firstDelay = 60000 } = {}) {
|
||||
let child = null;
|
||||
const run = () => {
|
||||
if (child) return;
|
||||
child = fork(script, [dataDir], {
|
||||
execArgv: ["--max-old-space-size=1024"],
|
||||
stdio: "inherit",
|
||||
});
|
||||
child.on("exit", () => (child = null));
|
||||
};
|
||||
const timers = [setTimeout(run, firstDelay), setInterval(run, 6 * HOUR)];
|
||||
for (const timer of timers) timer.unref();
|
||||
return () => {
|
||||
for (const timer of timers) clearInterval(timer);
|
||||
child?.kill();
|
||||
};
|
||||
}
|
||||
398
server/products/usda.js
Normal file
398
server/products/usda.js
Normal file
|
|
@ -0,0 +1,398 @@
|
|||
import { createWriteStream } from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import yauzl from "yauzl";
|
||||
import { databaseGtin } from "../../shared/barcode.js";
|
||||
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
|
||||
|
||||
export const USDA_PAGE = "https://fdc.nal.usda.gov/download-datasets";
|
||||
const archiveName =
|
||||
/FoodData_Central_branded_food_csv_(\d{4}-\d{2}-\d{2})\.zip/g;
|
||||
|
||||
/** The newest Branded Foods CSV archive linked from the download page. */
|
||||
export async function latestUsdaArchive(fetchImpl = fetch) {
|
||||
const response = await fetchImpl(USDA_PAGE, {
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`USDA answered ${response.status}`);
|
||||
const versions = [...(await response.text()).matchAll(archiveName)].map(
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!versions.length) throw new Error("No USDA branded archive is listed");
|
||||
const version = versions.sort().at(-1);
|
||||
return {
|
||||
version,
|
||||
url: `https://fdc.nal.usda.gov/fdc-datasets/FoodData_Central_branded_food_csv_${version}.zip`,
|
||||
};
|
||||
}
|
||||
|
||||
export async function download(url, file, fetchImpl = fetch) {
|
||||
const response = await fetchImpl(url);
|
||||
if (!response.ok) throw new Error(`Download failed: ${response.status}`);
|
||||
await pipeline(Readable.fromWeb(response.body), createWriteStream(file));
|
||||
}
|
||||
|
||||
// FoodData Central nutrient IDs mapped to Calio keys. Unlisted mass nutrients
|
||||
// are kept under a slug of their USDA name.
|
||||
const nutrientIds = {
|
||||
1008: "energy-kcal",
|
||||
1003: "proteins",
|
||||
1004: "fat",
|
||||
1005: "carbohydrates",
|
||||
1079: "fiber",
|
||||
2000: "sugars",
|
||||
1093: "sodium",
|
||||
1258: "saturated-fat",
|
||||
1257: "trans-fat",
|
||||
1292: "monounsaturated-fat",
|
||||
1293: "polyunsaturated-fat",
|
||||
1253: "cholesterol",
|
||||
1235: "added-sugars",
|
||||
1086: "polyols",
|
||||
1082: "soluble-fiber",
|
||||
1084: "insoluble-fiber",
|
||||
1009: "starch",
|
||||
1010: "sucrose",
|
||||
1011: "glucose",
|
||||
1012: "fructose",
|
||||
1013: "lactose",
|
||||
1014: "maltose",
|
||||
1075: "galactose",
|
||||
1056: "sorbitol",
|
||||
1078: "xylitol",
|
||||
1055: "mannitol",
|
||||
1051: "water",
|
||||
1018: "alcohol",
|
||||
1057: "caffeine",
|
||||
1234: "taurine",
|
||||
1068: "beta-glucan",
|
||||
1181: "inositol",
|
||||
1106: "vitamin-a",
|
||||
1104: "vitamin-a-iu",
|
||||
1162: "vitamin-c",
|
||||
1114: "vitamin-d",
|
||||
1109: "vitamin-e",
|
||||
1185: "vitamin-k",
|
||||
1165: "vitamin-b1",
|
||||
1166: "vitamin-b2",
|
||||
1167: "vitamin-pp",
|
||||
1175: "vitamin-b6",
|
||||
1190: "vitamin-b9",
|
||||
1177: "folates",
|
||||
1186: "folic-acid",
|
||||
1178: "vitamin-b12",
|
||||
1176: "biotin",
|
||||
1170: "pantothenic-acid",
|
||||
1180: "choline",
|
||||
1087: "calcium",
|
||||
1089: "iron",
|
||||
1092: "potassium",
|
||||
1090: "magnesium",
|
||||
1091: "phosphorus",
|
||||
1095: "zinc",
|
||||
1098: "copper",
|
||||
1101: "manganese",
|
||||
1103: "selenium",
|
||||
1100: "iodine",
|
||||
1096: "chromium",
|
||||
1102: "molybdenum",
|
||||
1088: "chloride",
|
||||
1099: "fluoride",
|
||||
};
|
||||
// Fallbacks used only when the preferred ID is absent.
|
||||
const fallbacks = {
|
||||
1062: ["energy-kcal", (kj) => kj / 4.184], // Energy, kJ
|
||||
1085: ["fat", (g) => g], // Total fat (NLEA)
|
||||
1063: ["sugars", (g) => g], // Sugars, Total
|
||||
1110: ["vitamin-d", (iu) => iu * 0.025], // Vitamin D, IU
|
||||
};
|
||||
const usdaUnits = { G: "g", MG: "mg", UG: "µg", KCAL: "kcal", IU: "IU" };
|
||||
const slug = (name) =>
|
||||
name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 48);
|
||||
|
||||
/** Builds a converter from a nutrient row (id, amount) to [key, value]. */
|
||||
export function nutrientMapper(nutrientRows) {
|
||||
const map = new Map();
|
||||
for (const { id, name, unit } of nutrientRows) {
|
||||
const from = usdaUnits[unit];
|
||||
let key = nutrientIds[id];
|
||||
if (fallbacks[id]) {
|
||||
const [target, convert] = fallbacks[id];
|
||||
map.set(id, { key: target, convert, fallback: true });
|
||||
continue;
|
||||
}
|
||||
if (!key) {
|
||||
if (!["g", "mg", "µg"].includes(from) || /do not use/i.test(name))
|
||||
continue;
|
||||
key = slug(name);
|
||||
if (!key) continue;
|
||||
}
|
||||
const to = nutrientUnit(key);
|
||||
const convert =
|
||||
from === to ? (value) => value : (value) => convertMass(value, from, to);
|
||||
if (from !== to && convertMass(1, from, to) === null) continue;
|
||||
map.set(id, { key, convert, fallback: false });
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Parses one record starting at `pos`; null when `text` ends mid-record.
|
||||
function parseRecord(text, pos) {
|
||||
const fields = [];
|
||||
for (;;) {
|
||||
if (pos >= text.length) return null;
|
||||
if (text[pos] === '"') {
|
||||
let value = "";
|
||||
let i = pos + 1;
|
||||
for (;;) {
|
||||
const quote = text.indexOf('"', i);
|
||||
if (quote === -1 || quote + 1 >= text.length) return null;
|
||||
value += text.slice(i, quote);
|
||||
if (text[quote + 1] !== '"') {
|
||||
pos = quote + 1;
|
||||
break;
|
||||
}
|
||||
value += '"'; // "" inside a quoted field
|
||||
i = quote + 2;
|
||||
}
|
||||
fields.push(value);
|
||||
} else {
|
||||
let end = pos;
|
||||
while (end < text.length && text[end] !== "," && text[end] !== "\n")
|
||||
end++;
|
||||
if (end >= text.length) return null;
|
||||
fields.push(text.slice(pos, end).replace(/\r$/, ""));
|
||||
pos = end;
|
||||
}
|
||||
const c = text[pos];
|
||||
if (c === ",") pos++;
|
||||
else if (c === "\n") return { fields, next: pos + 1 };
|
||||
else if (c === "\r" && text[pos + 1] === "\n")
|
||||
return { fields, next: pos + 2 };
|
||||
else if (pos + 1 >= text.length) return null;
|
||||
else pos++; // stray character after a closing quote
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal RFC 4180 reader: yields one array of fields per record. */
|
||||
export async function* csvRecords(stream) {
|
||||
stream.setEncoding("utf8");
|
||||
let buffer = "";
|
||||
for await (const chunk of stream) {
|
||||
buffer += chunk;
|
||||
let start = 0;
|
||||
for (let record; (record = parseRecord(buffer, start)); start = record.next)
|
||||
yield record.fields;
|
||||
buffer = buffer.slice(start);
|
||||
}
|
||||
if (buffer.trim()) {
|
||||
const record = parseRecord(buffer + "\n", 0);
|
||||
if (record) yield record.fields;
|
||||
}
|
||||
}
|
||||
|
||||
function openZip(file) {
|
||||
return new Promise((resolve, reject) =>
|
||||
yauzl.open(file, { lazyEntries: false, autoClose: false }, (error, zip) => {
|
||||
if (error) return reject(error);
|
||||
const entries = new Map();
|
||||
zip.on("entry", (entry) =>
|
||||
entries.set(entry.fileName.split("/").at(-1), entry),
|
||||
);
|
||||
zip.on("end", () => resolve({ zip, entries }));
|
||||
zip.on("error", reject);
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Yields the requested columns of each row, in the order asked for.
|
||||
async function* table(zip, entries, name, columns) {
|
||||
const entry = entries.get(name);
|
||||
if (!entry) throw new Error(`USDA archive is missing ${name}`);
|
||||
const stream = await new Promise((resolve, reject) =>
|
||||
zip.openReadStream(entry, (error, s) =>
|
||||
error ? reject(error) : resolve(s),
|
||||
),
|
||||
);
|
||||
let indexes;
|
||||
for await (const fields of csvRecords(stream)) {
|
||||
if (!indexes) {
|
||||
indexes = columns.map((column) => fields.indexOf(column));
|
||||
const missing = columns.filter((_, i) => indexes[i] < 0);
|
||||
if (missing.length)
|
||||
throw new Error(`USDA ${name} is missing ${missing.join(", ")}`);
|
||||
continue;
|
||||
}
|
||||
yield indexes.map((i) => fields[i]);
|
||||
}
|
||||
}
|
||||
|
||||
const servingUnits = { g: "g", grm: "g", ml: "ml", mlt: "ml" };
|
||||
function servingText(household, qty, unit) {
|
||||
const amount = `${+qty.toFixed(1)} ${unit}`;
|
||||
if (!household || /^(amount|per) /i.test(household) || !/\d/.test(household))
|
||||
return amount;
|
||||
// Some submissions already include the weight: "3/4 cup (20g)".
|
||||
return /\d\s*(g|ml|grams?)\b/i.test(household)
|
||||
? household
|
||||
: `${household} (${amount})`;
|
||||
}
|
||||
|
||||
/** Reads a Branded Foods CSV archive into `sink`. */
|
||||
export async function importUsdaZip(file, sink, { onProgress } = {}) {
|
||||
const { zip, entries } = await openZip(file);
|
||||
try {
|
||||
const nutrientRows = [];
|
||||
for await (const [id, name, unit] of table(zip, entries, "nutrient.csv", [
|
||||
"id",
|
||||
"name",
|
||||
"unit_name",
|
||||
]))
|
||||
nutrientRows.push({ id: Number(id), name, unit });
|
||||
const mapper = nutrientMapper(nutrientRows);
|
||||
|
||||
// A GTIN can appear several times as a product is resubmitted; keep the
|
||||
// newest submission.
|
||||
const byGtin = new Map();
|
||||
const strings = new Map();
|
||||
// Substrings keep their whole parse buffer alive; store fresh copies,
|
||||
// shared across the many products with the same brand or serving.
|
||||
const keep = (text) => {
|
||||
if (!text) return "";
|
||||
let copy = strings.get(text);
|
||||
if (copy === undefined) {
|
||||
copy = Buffer.from(text).toString();
|
||||
strings.set(copy, copy);
|
||||
}
|
||||
return copy;
|
||||
};
|
||||
for await (const [
|
||||
id,
|
||||
gtinUpc,
|
||||
brandName,
|
||||
brandOwner,
|
||||
servingSize,
|
||||
servingSizeUnit,
|
||||
householdText,
|
||||
available,
|
||||
modified,
|
||||
] of table(zip, entries, "branded_food.csv", [
|
||||
"fdc_id",
|
||||
"gtin_upc",
|
||||
"brand_name",
|
||||
"brand_owner",
|
||||
"serving_size",
|
||||
"serving_size_unit",
|
||||
"household_serving_fulltext",
|
||||
"available_date",
|
||||
"modified_date",
|
||||
])) {
|
||||
const gtin = databaseGtin(gtinUpc);
|
||||
if (!gtin) continue;
|
||||
const date = available || modified || "";
|
||||
const fdcId = Number(id);
|
||||
const current = byGtin.get(gtin);
|
||||
if (
|
||||
current &&
|
||||
(current.date > date ||
|
||||
(current.date === date && current.fdcId > fdcId))
|
||||
)
|
||||
continue;
|
||||
const qty = Number(servingSize);
|
||||
const household = (householdText || "").trim();
|
||||
const unit = servingUnits[(servingSizeUnit || "").toLowerCase()];
|
||||
byGtin.set(keep(gtin), {
|
||||
fdcId,
|
||||
date: keep(date),
|
||||
brand: keep((brandName || brandOwner || "").trim().slice(0, 100)),
|
||||
servingQty: unit && qty > 0 ? qty : null,
|
||||
servingUnit: unit || "g",
|
||||
serving:
|
||||
unit && qty > 0 ? keep(servingText(household, qty, unit)) : null,
|
||||
name: "",
|
||||
gtin: "",
|
||||
});
|
||||
}
|
||||
strings.clear();
|
||||
const byFdc = new Map();
|
||||
for (const [gtin, product] of byGtin) {
|
||||
product.gtin = gtin;
|
||||
byFdc.set(product.fdcId, product);
|
||||
}
|
||||
byGtin.clear();
|
||||
|
||||
for await (const [id, description] of table(zip, entries, "food.csv", [
|
||||
"fdc_id",
|
||||
"description",
|
||||
])) {
|
||||
const product = byFdc.get(Number(id));
|
||||
if (product)
|
||||
product.name = Buffer.from(description.slice(0, 200)).toString();
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
const finish = (product, values) => {
|
||||
const nutrients = {};
|
||||
const late = [];
|
||||
for (let i = 0; i < values.length; i += 2) {
|
||||
const mapping = mapper.get(values[i]);
|
||||
const number = values[i + 1];
|
||||
if (!mapping || !Number.isFinite(number) || number < 0) continue;
|
||||
if (mapping.fallback) late.push([mapping, number]);
|
||||
else {
|
||||
const value = mapping.convert(number);
|
||||
if (value !== null) nutrients[mapping.key] = value;
|
||||
}
|
||||
}
|
||||
for (const [mapping, number] of late)
|
||||
if (nutrients[mapping.key] === undefined)
|
||||
nutrients[mapping.key] = mapping.convert(number);
|
||||
if (
|
||||
nutrients["energy-kcal"] === undefined ||
|
||||
nutrients["energy-kcal"] > 950
|
||||
)
|
||||
return;
|
||||
sink.add({
|
||||
gtin: product.gtin,
|
||||
name: product.name,
|
||||
brand: product.brand,
|
||||
serving: product.serving,
|
||||
servingQty: product.servingQty,
|
||||
servingUnit: product.servingUnit,
|
||||
nutrients,
|
||||
});
|
||||
if (++count % 250000 === 0) onProgress?.(count);
|
||||
};
|
||||
// ~30 million rows, grouped by food: each product is finished as soon as
|
||||
// the next one starts, so only one food's nutrients are held at a time.
|
||||
let currentId = null;
|
||||
let values = [];
|
||||
for await (const [id, nutrientId, amount] of table(
|
||||
zip,
|
||||
entries,
|
||||
"food_nutrient.csv",
|
||||
["fdc_id", "nutrient_id", "amount"],
|
||||
)) {
|
||||
const fdcId = Number(id);
|
||||
if (fdcId !== currentId) {
|
||||
const done = byFdc.get(currentId);
|
||||
if (done) {
|
||||
finish(done, values);
|
||||
byFdc.delete(currentId);
|
||||
}
|
||||
currentId = fdcId;
|
||||
values = [];
|
||||
}
|
||||
if (byFdc.has(fdcId) && amount !== "")
|
||||
values.push(Number(nutrientId), Number(amount));
|
||||
}
|
||||
const last = byFdc.get(currentId);
|
||||
if (last) finish(last, values);
|
||||
return count;
|
||||
} finally {
|
||||
zip.close();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue