Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s

Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-22 13:41:44 -07:00
parent 8f6981c8a2
commit 80fc48b9f1
39 changed files with 4434 additions and 33 deletions

434
server/ocr/engine.js Normal file
View file

@ -0,0 +1,434 @@
// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors
// PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with
// unclip, then CTC decoding of each line crop.
import { readFileSync } from "node:fs";
import { join } from "node:path";
import ort from "onnxruntime-node";
import sharp from "sharp";
const DET_THRESH = 0.3;
const BOX_THRESH = 0.6;
const UNCLIP_RATIO = 1.5;
const REC_HEIGHT = 48;
const REC_BATCH = 8;
/** Reads the CTC character list out of a PaddleOCR inference.yml. */
export function readDictionary(yaml) {
const lines = yaml.split(/\r?\n/);
const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line));
if (start < 0) throw new Error("No character_dict in model config");
const chars = [];
for (const line of lines.slice(start + 1)) {
const match = /^\s*- (.*)$/.exec(line);
if (!match) break;
let value = match[1];
if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'");
else if (value.startsWith('"')) value = JSON.parse(value);
chars.push(value);
}
return ["", ...chars, " "]; // blank, dictionary, then the space character
}
// ---------- geometry ----------
function convexHull(points) {
points.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
const cross = (o, a, b) =>
(a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]);
const lower = [];
for (const p of points) {
while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0)
lower.pop();
lower.push(p);
}
const upper = [];
for (let i = points.length - 1; i >= 0; i--) {
const p = points[i];
while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0)
upper.pop();
upper.push(p);
}
return lower.slice(0, -1).concat(upper.slice(0, -1));
}
/** Minimum-area rectangle as centre, unit axes and side lengths. */
export function minAreaRect(points) {
const hull = convexHull(points);
if (hull.length < 3) {
const [x, y] = hull[0] || [0, 0];
return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 };
}
let best = null;
for (let i = 0; i < hull.length; i++) {
const [x1, y1] = hull[i];
const [x2, y2] = hull[(i + 1) % hull.length];
const length = Math.hypot(x2 - x1, y2 - y1);
if (!length) continue;
const ux = (x2 - x1) / length;
const uy = (y2 - y1) / length;
let minU = Infinity;
let maxU = -Infinity;
let minV = Infinity;
let maxV = -Infinity;
for (const [x, y] of hull) {
const u = x * ux + y * uy;
const v = -x * uy + y * ux;
minU = Math.min(minU, u);
maxU = Math.max(maxU, u);
minV = Math.min(minV, v);
maxV = Math.max(maxV, v);
}
const area = (maxU - minU) * (maxV - minV);
if (!best || area < best.area) {
const cu = (minU + maxU) / 2;
const cv = (minV + maxV) / 2;
best = {
area,
cx: cu * ux - cv * uy,
cy: cu * uy + cv * ux,
ux,
uy,
w: maxU - minU,
h: maxV - minV,
};
}
}
return best;
}
/**
* Orients a rectangle for reading: the axis nearer horizontal runs left to
* right, the other top to bottom. Returns the top-left corner and axes.
*/
function readingFrame({ cx, cy, ux, uy, w, h }) {
let ax = [ux, uy, w];
let bx = [-uy, ux, h];
if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax];
if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]];
if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]];
const [dx, dy, width] = ax;
const [ex, ey, height] = bx;
return {
x: cx - (dx * width) / 2 - (ex * height) / 2,
y: cy - (dy * width) / 2 - (ey * height) / 2,
dx,
dy,
ex,
ey,
width,
height,
};
}
// ---------- detection ----------
function detectionBoxes(prob, width, height) {
const visited = new Uint8Array(width * height);
const boxes = [];
const stack = [];
for (let start = 0; start < prob.length; start++) {
if (visited[start] || prob[start] <= DET_THRESH) continue;
// Flood-fill one connected region, keeping each row's extreme pixels as
// hull candidates.
const rows = new Map();
let sum = 0;
let count = 0;
stack.push(start);
visited[start] = 1;
while (stack.length) {
const index = stack.pop();
const x = index % width;
const y = (index - x) / width;
sum += prob[index];
count++;
const row = rows.get(y);
if (!row) rows.set(y, [x, x]);
else {
if (x < row[0]) row[0] = x;
if (x > row[1]) row[1] = x;
}
for (const next of [
x > 0 ? index - 1 : -1,
x < width - 1 ? index + 1 : -1,
y > 0 ? index - width : -1,
y < height - 1 ? index + width : -1,
])
if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) {
visited[next] = 1;
stack.push(next);
}
}
if (count < 4 || sum / count < BOX_THRESH) continue;
const points = [];
for (const [y, [x0, x1]] of rows)
points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]);
const rect = minAreaRect(points);
if (Math.min(rect.w, rect.h) < 3) continue;
// Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper.
const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h));
rect.w += 2 * d;
rect.h += 2 * d;
if (Math.min(rect.w, rect.h) < 5) continue;
boxes.push({ ...rect, score: sum / count });
}
return boxes;
}
// ---------- recognition ----------
/** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */
function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) {
const { data, width, height } = image;
const plane = REC_HEIGHT * batchWidth;
const scale = frame.height / REC_HEIGHT;
const taps = Math.min(4, Math.max(1, Math.round(scale)));
const pixel = (x, y, c) => {
const xi = Math.min(width - 1, Math.max(0, x));
const yi = Math.min(height - 1, Math.max(0, y));
return data[(yi * width + xi) * 3 + c];
};
for (let ty = 0; ty < REC_HEIGHT; ty++)
for (let tx = 0; tx < cropWidth; tx++) {
let r = 0;
let g = 0;
let b = 0;
for (let sy = 0; sy < taps; sy++)
for (let sx = 0; sx < taps; sx++) {
const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width;
const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height;
const x = frame.x + frame.dx * u + frame.ex * v - 0.5;
const y = frame.y + frame.dy * u + frame.ey * v - 0.5;
const x0 = Math.floor(x);
const y0 = Math.floor(y);
const fx = x - x0;
const fy = y - y0;
for (let c = 0; c < 3; c++) {
const value =
pixel(x0, y0, c) * (1 - fx) * (1 - fy) +
pixel(x0 + 1, y0, c) * fx * (1 - fy) +
pixel(x0, y0 + 1, c) * (1 - fx) * fy +
pixel(x0 + 1, y0 + 1, c) * fx * fy;
if (c === 0) r += value;
else if (c === 1) g += value;
else b += value;
}
}
const n = taps * taps * 127.5;
const at = offset + ty * batchWidth + tx;
out[at] = b / n - 1;
out[at + plane] = g / n - 1;
out[at + 2 * plane] = r / n - 1;
}
}
function ctcDecode(output, dims, chars) {
const [count, steps, classes] = dims;
const results = [];
for (let n = 0; n < count; n++) {
let text = "";
let confidence = 0;
let kept = 0;
let previous = -1;
for (let t = 0; t < steps; t++) {
const base = (n * steps + t) * classes;
let best = 0;
let bestValue = output[base];
for (let c = 1; c < classes; c++)
if (output[base + c] > bestValue) {
best = c;
bestValue = output[base + c];
}
if (best !== 0 && best !== previous) {
text += chars[best] ?? "";
confidence += bestValue;
kept++;
}
previous = best;
}
results.push({ text, score: kept ? confidence / kept : 0 });
}
return results;
}
// ---------- pipeline ----------
export async function createOcr(modelDir, { detSide = 1600 } = {}) {
const options = {
graphOptimizationLevel: "all",
// The arena would keep ~2 GB of detection buffers after the first scan.
enableCpuMemArena: false,
};
const [det, rec] = await Promise.all([
ort.InferenceSession.create(
join(modelDir, "det", "inference.onnx"),
options,
),
ort.InferenceSession.create(
join(modelDir, "rec", "inference.onnx"),
options,
),
]);
const chars = readDictionary(
readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"),
);
async function detect(image) {
// Small photos are enlarged too: tiny print detects far better.
const ratio = Math.min(2, detSide / Math.max(image.width, image.height));
const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32);
const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
const { data } = await sharp(image.data, {
raw: { width: image.width, height: image.height, channels: 3 },
})
.resize(w, h, { fit: "fill" })
.raw()
.toBuffer({ resolveWithObject: true });
const input = new Float32Array(3 * w * h);
const mean = [0.406, 0.456, 0.485]; // applied to B, G, R
const std = [0.225, 0.224, 0.229];
for (let i = 0; i < w * h; i++)
for (let c = 0; c < 3; c++)
// Channel c of the tensor is BGR; data is RGB.
input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c];
const result = await det.run({
[det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]),
});
const prob = result[det.outputNames[0]].data;
const sx = image.width / w;
const sy = image.height / h;
return detectionBoxes(prob, w, h).map((box) => {
// Scale the rectangle back to the full image.
const ax = box.ux * sx;
const ay = box.uy * sy;
const length = Math.hypot(ax, ay);
const bx = -box.uy * sx;
const by = box.ux * sy;
return {
cx: box.cx * sx,
cy: box.cy * sy,
ux: ax / length,
uy: ay / length,
w: box.w * length,
h: box.h * Math.hypot(bx, by),
score: box.score,
};
});
}
async function recognize(image, boxes) {
const frames = boxes.map((box) => {
const frame = readingFrame(box);
return { frame, ratio: frame.width / Math.max(1, frame.height) };
});
const order = frames
.map((_, i) => i)
.sort((a, b) => frames[a].ratio - frames[b].ratio);
const texts = new Array(frames.length);
for (let start = 0; start < order.length; start += REC_BATCH) {
const batch = order.slice(start, start + REC_BATCH);
const maxRatio = Math.max(
320 / REC_HEIGHT,
...batch.map((i) => frames[i].ratio),
);
const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio));
const plane = REC_HEIGHT * batchWidth;
const input = new Float32Array(batch.length * 3 * plane);
batch.forEach((i, n) => {
const cropWidth = Math.min(
batchWidth,
Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)),
);
sampleCrop(
image,
frames[i].frame,
input,
n * 3 * plane,
batchWidth,
cropWidth,
);
});
const result = await rec.run({
[rec.inputNames[0]]: new ort.Tensor("float32", input, [
batch.length,
3,
REC_HEIGHT,
batchWidth,
]),
});
const output = result[rec.outputNames[0]];
ctcDecode(output.data, output.dims, chars).forEach((value, n) => {
texts[batch[n]] = value;
});
}
return boxes.map((box, i) => {
const { frame } = frames[i];
const xs = [
frame.x,
frame.x + frame.dx * frame.width,
frame.x + frame.ex * frame.height,
frame.x + frame.dx * frame.width + frame.ex * frame.height,
];
const ys = [
frame.y,
frame.y + frame.dy * frame.width,
frame.y + frame.ey * frame.height,
frame.y + frame.dy * frame.width + frame.ey * frame.height,
];
return {
text: texts[i].text,
score: texts[i].score,
angle: Math.atan2(frame.dy, frame.dx),
cx: (Math.min(...xs) + Math.max(...xs)) / 2,
cy: (Math.min(...ys) + Math.max(...ys)) / 2,
x0: Math.min(...xs),
y0: Math.min(...ys),
x1: Math.max(...xs),
y1: Math.max(...ys),
height: frame.height,
};
});
}
async function decode(buffer, rotate = 0) {
const { data, info } = await sharp(buffer, { failOn: "none" })
.rotate() // honour EXIF orientation
.rotate(rotate)
.resize(2400, 2400, { fit: "inside", withoutEnlargement: true })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
return { data, width: info.width, height: info.height };
}
async function readImage(image) {
const boxes = await detect(image);
const lines = (await recognize(image, boxes)).filter((line) =>
line.text.trim(),
);
const score = lines.reduce(
(sum, line) => sum + line.score * line.text.length,
0,
);
return { lines, score, boxes };
}
/** Returns recognised lines with their bounding boxes, in image pixels. */
async function read(buffer) {
const image = await decode(buffer);
let best = await readImage(image);
// A label photographed sideways yields mostly tall, narrow boxes; try
// both quarter turns and keep whichever reads more confidently.
const tall = best.boxes.filter((box) => {
const frame = readingFrame(box);
return frame.height > frame.width * 1.5;
}).length;
if (best.boxes.length && tall > best.boxes.length / 2)
for (const angle of [90, 270]) {
const attempt = await readImage(await decode(buffer, angle));
if (attempt.score > best.score) best = attempt;
}
return best.lines;
}
return { read };
}

461
server/ocr/label.js Normal file
View file

@ -0,0 +1,461 @@
// Turns OCR text boxes from a Nutrition Facts panel into a food. Built for
// US/Canadian labels, with the common EU "per 100 g" table as a fallback.
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
// FDA Daily Values (2016 rule) in the unit printed on labels; a %DV next to
// an amount is used to catch misread digits and to fill amounts that are only
// given as a percentage.
const dailyValues = {
fat: [78, "g"],
"saturated-fat": [20, "g"],
cholesterol: [300, "mg"],
sodium: [2300, "mg"],
carbohydrates: [275, "g"],
fiber: [28, "g"],
"added-sugars": [50, "g"],
proteins: [50, "g"],
"vitamin-d": [20, "mcg"],
calcium: [1300, "mg"],
iron: [18, "mg"],
potassium: [4700, "mg"],
"vitamin-a": [900, "mcg"],
"vitamin-c": [90, "mg"],
"vitamin-e": [15, "mg"],
"vitamin-k": [120, "mcg"],
"vitamin-b1": [1.2, "mg"],
"vitamin-b2": [1.3, "mg"],
"vitamin-pp": [16, "mg"],
"vitamin-b6": [1.7, "mg"],
"vitamin-b9": [400, "mcg"],
"vitamin-b12": [2.4, "mcg"],
biotin: [30, "mcg"],
"pantothenic-acid": [5, "mg"],
phosphorus: [1250, "mg"],
iodine: [150, "mcg"],
magnesium: [420, "mg"],
zinc: [11, "mg"],
selenium: [55, "mcg"],
copper: [0.9, "mg"],
manganese: [2.3, "mg"],
chromium: [35, "mcg"],
molybdenum: [45, "mcg"],
chloride: [2300, "mg"],
choline: [550, "mg"],
};
// Most specific first: "saturated fat" must claim its row before "fat" does.
const fields = [
["trans-fat", /\btrans\.?\s*fat|\bgrasas?\s*trans\b/],
[
"saturated-fat",
/\b(?:of which\s*)?sat(?:urated|\.)?\s*fat|\bsaturates\b|\bgrasas?\s*saturadas?\b/,
],
["polyunsaturated-fat", /\bpoly(?:unsat(?:urated|\.)?)?\s*fat/],
["monounsaturated-fat", /\bmono(?:unsat(?:urated|\.)?)?\s*fat/],
[
"fat",
/\b(?:total\s*)?fat\b(?!\s*free)|\bgrasas?(?:\s*totales?)?(?:\s*\(?lipidos\)?)?/,
],
["cholesterol", /\bcholest(?:erol|\.)?|\bcolesterol\b/],
["sodium", /\bsodium\b|\bsodio\b/],
["salt", /\bsalt\b/],
["soluble-fiber", /(?<!in)\bsoluble\s*fib(?:er|re)/],
["insoluble-fiber", /\binsoluble\s*fib(?:er|re)/],
["fiber", /\b(?:dietary\s*)?fib(?:er|re)\b|\bfibra(?:\s*dietetica)?\b/],
["polyols", /\bsugar\s*alcohols?\b|\bpolyols\b/],
[
"sugars",
/\b(?:total\s*|of which\s*)?sugars?\b|\bazucares(?:\s*totales)?\b(?!\s*anadidos)/,
],
[
"carbohydrates",
/\b(?:total\s*)?carb(?:ohydrates?|s|\.)?|\bcarbohidratos(?:\s*\(?hidratos\s*(?:\d\s*)?de\s*carbono\)?)?|\bhidratos\s*de\s*carbono\b/,
],
["proteins", /\bprotein\b|\bproteinas?\b/],
["vitamin-d", /\bvit(?:amin|amina|\.)?\s*d\d?\b/],
["vitamin-a", /\bvit(?:amin|\.)?\s*a\b/],
["vitamin-c", /\bvit(?:amin|\.)?\s*c\b/],
["vitamin-e", /\bvit(?:amin|\.)?\s*e\b/],
["vitamin-k", /\bvit(?:amin|\.)?\s*k\d?\b/],
["vitamin-b6", /\bvit(?:amin|\.)?\s*b\s*-?6\b/],
["vitamin-b12", /\bvit(?:amin|\.)?\s*b\s*-?12\b/],
["vitamin-b1", /\bthiamine?\b/],
["vitamin-b2", /\briboflavin\b/],
["vitamin-pp", /\bniacin\b/],
["vitamin-b9", /\bfolate\b|\bfolic acid\b/],
["biotin", /\bbiotin\b/],
["pantothenic-acid", /\bpantothenic\b/],
["calcium", /\bcalcium\b|\bcalcio\b/],
["iron", /\biron\b|\bhierro\b/],
["potassium", /\bpotas(?:sium|\.)?|\bpotasio\b/],
["phosphorus", /\bphosphorus\b/],
["magnesium", /\bmagnesium\b/],
["zinc", /\bzinc\b/],
["selenium", /\bselenium\b/],
["copper", /\bcopper\b/],
["manganese", /\bmanganese\b/],
["iodine", /\biodine\b/],
["chromium", /\bchromium\b/],
["molybdenum", /\bmolybdenum\b/],
["chloride", /\bchloride\b/],
["choline", /\bcholine\b/],
["caffeine", /\bcaffeine\b/],
];
// The unit printed when OCR loses it.
const defaultUnits = {
"energy-kcal": "kcal",
sodium: "mg",
cholesterol: "mg",
potassium: "mg",
calcium: "mg",
iron: "mg",
"vitamin-d": "mcg",
};
/**
* Groups boxes into reading rows. Coordinates are first turned by the
* dominant text angle, so a tilted photo still lines up "Sodium 160mg 7%".
*/
export function rows(lines) {
const angles = lines
.filter(
(line) => line.angle !== undefined && line.x1 - line.x0 > 3 * line.height,
)
.map((line) => line.angle)
.sort((a, b) => a - b);
const angle = angles.length ? angles[Math.floor(angles.length / 2)] : 0;
const cos = Math.cos(angle);
const sin = Math.sin(angle);
const placed = lines.map((line) => {
const cx = line.cx ?? (line.x0 + line.x1) / 2;
const cy = line.cy ?? (line.y0 + line.y1) / 2;
const height = line.height ?? line.y1 - line.y0;
const y = -cx * sin + cy * cos;
return {
text: line.text,
x: cx * cos + cy * sin,
y0: y - height / 2,
y1: y + height / 2,
};
});
placed.sort((a, b) => a.y0 + a.y1 - b.y0 - b.y1);
const result = [];
for (const line of placed) {
const height = line.y1 - line.y0;
const row = result.find((candidate) => {
const overlap =
Math.min(candidate.y1, line.y1) - Math.max(candidate.y0, line.y0);
return overlap > 0.5 * Math.min(height, candidate.y1 - candidate.y0);
});
if (row) row.items.push(line);
else result.push({ y0: line.y0, y1: line.y1, items: [line] });
}
return result
.sort((a, b) => a.y0 - b.y0)
.map((row) => {
row.items.sort((a, b) => a.x - b.x);
return clean(row.items.map((item) => item.text).join(" "));
});
}
// Label vocabulary; OCR misspellings within a small edit distance of these
// ("Sodlum", "Proteln", "Caicium") are snapped back before matching.
const vocabulary = `calories total fat saturated trans polyunsaturated
monounsaturated cholesterol sodium carbohydrate carbohydrates dietary fiber
soluble insoluble sugars sugar added includes alcohols protein vitamin
calcium iron potassium serving servings size container amount daily value
thiamin riboflavin niacin folate phosphorus magnesium zinc selenium copper
manganese biotin pantothenic iodine chromium molybdenum chloride choline
caffeine energy saturates fibre salt nutrition facts grasa grasas saturada
colesterol sodio fibra dietetica azucares anadidos carbohidratos hidratos
carbono proteinas calcio hierro potasio vitamina energetico contenido
porcion lipidos`.split(/\s+/);
const known = new Set(vocabulary);
function distance(a, b, limit) {
if (Math.abs(a.length - b.length) > limit) return limit + 1;
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
for (let i = 1; i <= a.length; i++) {
const current = [i];
let best = i;
for (let j = 1; j <= b.length; j++) {
current[j] = Math.min(
previous[j] + 1,
current[j - 1] + 1,
previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
);
best = Math.min(best, current[j]);
}
if (best > limit) return limit + 1;
previous = current;
}
return previous[b.length];
}
function correctWord(word) {
const lower = word.toLowerCase();
if (known.has(lower)) return word;
const limit = lower.length >= 8 ? 2 : 1;
let best = null;
let bestDistance = limit + 1;
for (const candidate of vocabulary) {
const d = distance(lower, candidate, limit);
if (d < bestDistance) {
best = candidate;
bestDistance = d;
}
}
return best ?? word;
}
/** Undoes the usual OCR confusions in label text. */
export function clean(text) {
return (
text
.normalize("NFD")
.replace(/[̀-ͯ]/g, "") // Azúcares → Azucares
.replace(/[a-z]{5,}/gi, correctWord)
.replace(/[µμ]g\b/gi, "mcg")
.replace(/\bug\b/gi, "mcg")
.replace(/less than/gi, "<")
// Letters standing in for digits next to digits or units: "Og", "1O".
.replace(/(?<=\d)[oO]|[oO](?=\d)/g, "0")
.replace(/(?<![a-z])[oO](?=\s?(?:m?c?g|%)\b)/g, "0")
.replace(/(?<=\d)[lI|](?=\d|\s?m?g\b)|(?<![a-z])[lI|](?=\d)/g, "1")
// A footnote mark read as a digit: "Carbohydrate1 13g".
.replace(/\b([a-z]{4,})[1†*]\s+(?=<?\d)/gi, "$1 ")
// Thousands separators, then decimal commas.
.replace(/(\d),(\d{3})(?!\d)/g, "$1$2")
.replace(/(\d),(\d)/g, "$1.$2")
.replace(/\s+/g, " ")
.trim()
);
}
// The amount may follow a split word ("Carbohy ydrate 42g"); a number
// directly followed by % is a percentage, not an amount.
const amountPattern =
/^\s*((?:[a-z]{1,10}\s+){0,2})[\s:.\-–—]*(<\s*)?(\d+(?:\.\d+)?|\.\d+)\s*(mcg|mg|g|kcal|kj|iu)?(?![a-z%\d.])(?!\s*%)/i;
const percentPattern = /^[^\d%]{0,12}?(\d{1,4})\s*%/;
function readAmount(rest) {
const found = amountPattern.exec(rest);
if (!found) return null;
// Never skip over another nutrient's name: in "Cholest. Protein 9g" the
// 9g is protein's.
const skipped = found[1].trim().toLowerCase().split(/\s+/).filter(Boolean);
if (skipped.some((word) => known.has(word))) return null;
const match = [found[0], ...found.slice(2)];
const after = rest.slice(match[0].length);
const percent = percentPattern.exec(after);
return {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3]?.toLowerCase(),
percent: percent ? Number(percent[1]) : null,
};
}
const toCatalog = (key, value, unit) =>
key === "energy-kcal"
? value
: convertMass(value, unit === "mcg" ? "µg" : unit, nutrientUnit(key));
/** Resolves an amount, using its %DV to correct misreads such as 8g → "89". */
function resolve(key, reading) {
const dv = dailyValues[key];
let unit = reading.unit || defaultUnits[key] || dv?.[1] || "g";
if (unit === "iu" || unit === "kj" || unit === "kcal") return null;
if (unit === "g" && dv?.[1] === "mcg") unit = "mcg"; // "mcg" read as "g"
let value = reading.less ? reading.value / 2 : reading.value;
if (dv && reading.percent !== null) {
const daily = convertMass(dv[0], mass(dv[1]), mass(unit));
// %DV is rounded (and may read 0 below 2%), so compare in points.
const consistent = (candidate) =>
Math.abs((candidate / daily) * 100 - reading.percent) <=
Math.max(2, reading.percent * 0.3);
if (daily && !consistent(value)) {
const candidates = [];
// A unit read as a digit: "8g" → "89", "160mg" → "16019".
if (!reading.unit && /\d9$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -1)));
if (!reading.unit && /\d19$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -2)));
// A lost decimal point: "05" → 0.5, "25" → 2.5.
if (/^\d{2,3}$/.test(reading.raw))
candidates.push(Number(reading.raw) / 10);
const fixed = candidates.find(consistent);
if (fixed !== undefined) value = fixed;
// Nothing plausible was read; trust the percentage.
else if (!reading.unit) value = (reading.percent / 100) * daily;
}
}
const result = toCatalog(key, value, unit);
// Far beyond a day's worth in one serving is a misread, not a food.
if (dv && result > 10 * toCatalog(key, dv[0], dv[1])) return null;
return result;
}
const mass = (unit) => (unit === "mcg" ? "µg" : unit);
function percentOnly(key, rest) {
const dv = dailyValues[key];
const match = /^[^\d]{0,6}(\d{1,4})\s*%/.exec(rest);
if (!dv || !match) return null;
return toCatalog(key, (Number(match[1]) / 100) * dv[0], dv[1]);
}
/**
* @param {{text: string, x0: number, y0: number, x1: number, y1: number}[]} lines
* @returns {{serving?: string, nutrients: Record<string, number>,
* extra: Record<string, number>, found: string[], text: string[]}}
*/
export function parseLabel(lines) {
const text = rows(lines);
const values = {};
const claimed = new Set(); // character spans already used, per row
const used = (row, start, end) => {
for (let i = start; i < end; i++)
if (claimed.has(`${row}:${i}`)) return true;
for (let i = start; i < end; i++) claimed.add(`${row}:${i}`);
return false;
};
// "Includes 10g Added Sugars" puts the amount before the name.
text.forEach((row, index) => {
const match =
/\bincl(?:udes|\.)?\s*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?\s*(?:of\s*)?added\s*sugars?/i.exec(
row,
) ||
/\b(?:added\s*sugars?|azucares\s*anadidos)[\s:]*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?/i.exec(
row,
);
if (!match || values["added-sugars"] !== undefined) return;
used(index, match.index, match.index + match[0].length);
const percent = percentPattern.exec(
row.slice(match.index + match[0].length),
);
values["added-sugars"] = resolve("added-sugars", {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3],
percent: percent ? Number(percent[1]) : null,
});
});
for (const [key, pattern] of fields) {
if (values[key] !== undefined) continue;
for (
let index = 0;
index < text.length && values[key] === undefined;
index++
) {
const regex = new RegExp(pattern.source, "gi");
for (let match; (match = regex.exec(text[index])); ) {
const start = match.index;
const end = start + match[0].length;
if (used(index, start, end)) continue;
let rest = text[index].slice(end);
// A linear label can wrap between a name and its amount.
if (
/^[\s,.:]*$/.test(rest) &&
/^<?\s*\.?\d/.test(text[index + 1] || "")
)
rest = " " + text[index + 1];
const reading = readAmount(rest);
const value = reading ? resolve(key, reading) : percentOnly(key, rest);
if (value === null || value === undefined) continue;
values[key] = value;
break;
}
}
}
if (values.salt !== undefined) {
if (values.sodium === undefined) values.sodium = (values.salt / 2.5) * 1000;
delete values.salt;
}
// Calories: the amount follows on the same row, or sits alone on a
// neighbouring row when it's printed in a much larger font.
for (let index = 0; index < text.length; index++) {
const row = text[index];
const match =
/\bcalories\b(?!\s*from)|\benergy\b|\benerg(?:ia|etico)\b/i.exec(row);
if (!match) continue;
const rest = row.slice(match.index + match[0].length);
const kcal =
/(\d+(?:\.\d+)?)\s*kcal/i.exec(rest) ||
/^[^\d]{0,24}?(\d{1,4})(?!\s*(?:k?j|%|m?c?g))\b/i.exec(rest);
const before = text[index - 1] || "";
const after = text[index + 1] || "";
const perServing =
/amount\s*per\s*serving[:\s]*(\d{1,4})\b(?!\s*(?:m?c?g|%))/i;
const alone =
// Big type often lands beside "Amount per serving" on the next row.
perServing.exec(before) ||
perServing.exec(after) ||
// A number alone at the end of a neighbouring row.
[after, before]
.map((neighbour) =>
/%|\d\s*m?c?g\b/i.test(neighbour)
? null
: /(?:^|\s)(\d{1,4})$/.exec(neighbour),
)
.find(Boolean) ||
// A linear label wrapped right after "Calories": "80, Total Fat…".
/^(\d{1,4})\b(?!\s*(?:m?c?g|%|\.\d))/.exec(after);
const value = Number((kcal || alone)?.[1]);
if (Number.isFinite(value) && value <= 5000 && (kcal || alone)) {
values["energy-kcal"] = value;
break;
}
}
let serving;
const perHundred = text.some((row) => /\bper\s*100\s*(g|ml)\b/i.test(row));
for (let index = 0; index < text.length; index++) {
const match = /\bserv(?:ing|\.)\s*size\b[\s:]*(.*)$/i.exec(text[index]);
if (!match) continue;
let size = match[1].trim();
if (!/\d/.test(size)) size = `${size} ${text[index + 1] || ""}`.trim();
// Two-column layouts can run nutrient rows into the serving text.
size = size
.split(/\b(?:calories|amount|total|sodium|protein|servings)\b|%/i)[0]
.trim()
.replace(/\s*\(\s*/g, " (")
.replace(/\s*\)\s*/g, ")")
.replace(/(\d)\s*(g|ml)\b/gi, "$1 $2")
.replace(/\)\s*.*$/, ")")
.trim();
if (size && /\d/.test(size)) serving = size.slice(0, 100);
break;
}
if (!serving && perHundred) {
const ml = text.some((row) => /\bper\s*100\s*ml\b/i.test(row));
serving = ml ? "100 ml" : "100 g";
}
const core = {
"energy-kcal": "calories",
proteins: "protein",
carbohydrates: "carbs",
fat: "fat",
fiber: "fiber",
sugars: "sugar",
sodium: "sodium",
};
const nutrients = {};
const extra = {};
for (const [key, value] of Object.entries(values)) {
if (!Number.isFinite(value) || value < 0 || value > 100000) continue;
const rounded = Math.round(value * 1000) / 1000;
if (core[key]) nutrients[core[key]] = rounded;
else extra[key] = rounded;
}
return {
...(serving ? { serving } : {}),
nutrients,
extra,
found: Object.keys(values),
text,
};
}