Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s

Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-22 13:41:44 -07:00
parent 8f6981c8a2
commit 80fc48b9f1
39 changed files with 4434 additions and 33 deletions

461
server/ocr/label.js Normal file
View file

@ -0,0 +1,461 @@
// Turns OCR text boxes from a Nutrition Facts panel into a food. Built for
// US/Canadian labels, with the common EU "per 100 g" table as a fallback.
import { convertMass, nutrientUnit } from "../../shared/nutrients.js";
// FDA Daily Values (2016 rule) in the unit printed on labels; a %DV next to
// an amount is used to catch misread digits and to fill amounts that are only
// given as a percentage.
const dailyValues = {
fat: [78, "g"],
"saturated-fat": [20, "g"],
cholesterol: [300, "mg"],
sodium: [2300, "mg"],
carbohydrates: [275, "g"],
fiber: [28, "g"],
"added-sugars": [50, "g"],
proteins: [50, "g"],
"vitamin-d": [20, "mcg"],
calcium: [1300, "mg"],
iron: [18, "mg"],
potassium: [4700, "mg"],
"vitamin-a": [900, "mcg"],
"vitamin-c": [90, "mg"],
"vitamin-e": [15, "mg"],
"vitamin-k": [120, "mcg"],
"vitamin-b1": [1.2, "mg"],
"vitamin-b2": [1.3, "mg"],
"vitamin-pp": [16, "mg"],
"vitamin-b6": [1.7, "mg"],
"vitamin-b9": [400, "mcg"],
"vitamin-b12": [2.4, "mcg"],
biotin: [30, "mcg"],
"pantothenic-acid": [5, "mg"],
phosphorus: [1250, "mg"],
iodine: [150, "mcg"],
magnesium: [420, "mg"],
zinc: [11, "mg"],
selenium: [55, "mcg"],
copper: [0.9, "mg"],
manganese: [2.3, "mg"],
chromium: [35, "mcg"],
molybdenum: [45, "mcg"],
chloride: [2300, "mg"],
choline: [550, "mg"],
};
// Most specific first: "saturated fat" must claim its row before "fat" does.
const fields = [
["trans-fat", /\btrans\.?\s*fat|\bgrasas?\s*trans\b/],
[
"saturated-fat",
/\b(?:of which\s*)?sat(?:urated|\.)?\s*fat|\bsaturates\b|\bgrasas?\s*saturadas?\b/,
],
["polyunsaturated-fat", /\bpoly(?:unsat(?:urated|\.)?)?\s*fat/],
["monounsaturated-fat", /\bmono(?:unsat(?:urated|\.)?)?\s*fat/],
[
"fat",
/\b(?:total\s*)?fat\b(?!\s*free)|\bgrasas?(?:\s*totales?)?(?:\s*\(?lipidos\)?)?/,
],
["cholesterol", /\bcholest(?:erol|\.)?|\bcolesterol\b/],
["sodium", /\bsodium\b|\bsodio\b/],
["salt", /\bsalt\b/],
["soluble-fiber", /(?<!in)\bsoluble\s*fib(?:er|re)/],
["insoluble-fiber", /\binsoluble\s*fib(?:er|re)/],
["fiber", /\b(?:dietary\s*)?fib(?:er|re)\b|\bfibra(?:\s*dietetica)?\b/],
["polyols", /\bsugar\s*alcohols?\b|\bpolyols\b/],
[
"sugars",
/\b(?:total\s*|of which\s*)?sugars?\b|\bazucares(?:\s*totales)?\b(?!\s*anadidos)/,
],
[
"carbohydrates",
/\b(?:total\s*)?carb(?:ohydrates?|s|\.)?|\bcarbohidratos(?:\s*\(?hidratos\s*(?:\d\s*)?de\s*carbono\)?)?|\bhidratos\s*de\s*carbono\b/,
],
["proteins", /\bprotein\b|\bproteinas?\b/],
["vitamin-d", /\bvit(?:amin|amina|\.)?\s*d\d?\b/],
["vitamin-a", /\bvit(?:amin|\.)?\s*a\b/],
["vitamin-c", /\bvit(?:amin|\.)?\s*c\b/],
["vitamin-e", /\bvit(?:amin|\.)?\s*e\b/],
["vitamin-k", /\bvit(?:amin|\.)?\s*k\d?\b/],
["vitamin-b6", /\bvit(?:amin|\.)?\s*b\s*-?6\b/],
["vitamin-b12", /\bvit(?:amin|\.)?\s*b\s*-?12\b/],
["vitamin-b1", /\bthiamine?\b/],
["vitamin-b2", /\briboflavin\b/],
["vitamin-pp", /\bniacin\b/],
["vitamin-b9", /\bfolate\b|\bfolic acid\b/],
["biotin", /\bbiotin\b/],
["pantothenic-acid", /\bpantothenic\b/],
["calcium", /\bcalcium\b|\bcalcio\b/],
["iron", /\biron\b|\bhierro\b/],
["potassium", /\bpotas(?:sium|\.)?|\bpotasio\b/],
["phosphorus", /\bphosphorus\b/],
["magnesium", /\bmagnesium\b/],
["zinc", /\bzinc\b/],
["selenium", /\bselenium\b/],
["copper", /\bcopper\b/],
["manganese", /\bmanganese\b/],
["iodine", /\biodine\b/],
["chromium", /\bchromium\b/],
["molybdenum", /\bmolybdenum\b/],
["chloride", /\bchloride\b/],
["choline", /\bcholine\b/],
["caffeine", /\bcaffeine\b/],
];
// The unit printed when OCR loses it.
const defaultUnits = {
"energy-kcal": "kcal",
sodium: "mg",
cholesterol: "mg",
potassium: "mg",
calcium: "mg",
iron: "mg",
"vitamin-d": "mcg",
};
/**
* Groups boxes into reading rows. Coordinates are first turned by the
* dominant text angle, so a tilted photo still lines up "Sodium 160mg 7%".
*/
export function rows(lines) {
const angles = lines
.filter(
(line) => line.angle !== undefined && line.x1 - line.x0 > 3 * line.height,
)
.map((line) => line.angle)
.sort((a, b) => a - b);
const angle = angles.length ? angles[Math.floor(angles.length / 2)] : 0;
const cos = Math.cos(angle);
const sin = Math.sin(angle);
const placed = lines.map((line) => {
const cx = line.cx ?? (line.x0 + line.x1) / 2;
const cy = line.cy ?? (line.y0 + line.y1) / 2;
const height = line.height ?? line.y1 - line.y0;
const y = -cx * sin + cy * cos;
return {
text: line.text,
x: cx * cos + cy * sin,
y0: y - height / 2,
y1: y + height / 2,
};
});
placed.sort((a, b) => a.y0 + a.y1 - b.y0 - b.y1);
const result = [];
for (const line of placed) {
const height = line.y1 - line.y0;
const row = result.find((candidate) => {
const overlap =
Math.min(candidate.y1, line.y1) - Math.max(candidate.y0, line.y0);
return overlap > 0.5 * Math.min(height, candidate.y1 - candidate.y0);
});
if (row) row.items.push(line);
else result.push({ y0: line.y0, y1: line.y1, items: [line] });
}
return result
.sort((a, b) => a.y0 - b.y0)
.map((row) => {
row.items.sort((a, b) => a.x - b.x);
return clean(row.items.map((item) => item.text).join(" "));
});
}
// Label vocabulary; OCR misspellings within a small edit distance of these
// ("Sodlum", "Proteln", "Caicium") are snapped back before matching.
const vocabulary = `calories total fat saturated trans polyunsaturated
monounsaturated cholesterol sodium carbohydrate carbohydrates dietary fiber
soluble insoluble sugars sugar added includes alcohols protein vitamin
calcium iron potassium serving servings size container amount daily value
thiamin riboflavin niacin folate phosphorus magnesium zinc selenium copper
manganese biotin pantothenic iodine chromium molybdenum chloride choline
caffeine energy saturates fibre salt nutrition facts grasa grasas saturada
colesterol sodio fibra dietetica azucares anadidos carbohidratos hidratos
carbono proteinas calcio hierro potasio vitamina energetico contenido
porcion lipidos`.split(/\s+/);
const known = new Set(vocabulary);
function distance(a, b, limit) {
if (Math.abs(a.length - b.length) > limit) return limit + 1;
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
for (let i = 1; i <= a.length; i++) {
const current = [i];
let best = i;
for (let j = 1; j <= b.length; j++) {
current[j] = Math.min(
previous[j] + 1,
current[j - 1] + 1,
previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1),
);
best = Math.min(best, current[j]);
}
if (best > limit) return limit + 1;
previous = current;
}
return previous[b.length];
}
function correctWord(word) {
const lower = word.toLowerCase();
if (known.has(lower)) return word;
const limit = lower.length >= 8 ? 2 : 1;
let best = null;
let bestDistance = limit + 1;
for (const candidate of vocabulary) {
const d = distance(lower, candidate, limit);
if (d < bestDistance) {
best = candidate;
bestDistance = d;
}
}
return best ?? word;
}
/** Undoes the usual OCR confusions in label text. */
export function clean(text) {
return (
text
.normalize("NFD")
.replace(/[̀-ͯ]/g, "") // Azúcares → Azucares
.replace(/[a-z]{5,}/gi, correctWord)
.replace(/[µμ]g\b/gi, "mcg")
.replace(/\bug\b/gi, "mcg")
.replace(/less than/gi, "<")
// Letters standing in for digits next to digits or units: "Og", "1O".
.replace(/(?<=\d)[oO]|[oO](?=\d)/g, "0")
.replace(/(?<![a-z])[oO](?=\s?(?:m?c?g|%)\b)/g, "0")
.replace(/(?<=\d)[lI|](?=\d|\s?m?g\b)|(?<![a-z])[lI|](?=\d)/g, "1")
// A footnote mark read as a digit: "Carbohydrate1 13g".
.replace(/\b([a-z]{4,})[1†*]\s+(?=<?\d)/gi, "$1 ")
// Thousands separators, then decimal commas.
.replace(/(\d),(\d{3})(?!\d)/g, "$1$2")
.replace(/(\d),(\d)/g, "$1.$2")
.replace(/\s+/g, " ")
.trim()
);
}
// The amount may follow a split word ("Carbohy ydrate 42g"); a number
// directly followed by % is a percentage, not an amount.
const amountPattern =
/^\s*((?:[a-z]{1,10}\s+){0,2})[\s:.\-–—]*(<\s*)?(\d+(?:\.\d+)?|\.\d+)\s*(mcg|mg|g|kcal|kj|iu)?(?![a-z%\d.])(?!\s*%)/i;
const percentPattern = /^[^\d%]{0,12}?(\d{1,4})\s*%/;
function readAmount(rest) {
const found = amountPattern.exec(rest);
if (!found) return null;
// Never skip over another nutrient's name: in "Cholest. Protein 9g" the
// 9g is protein's.
const skipped = found[1].trim().toLowerCase().split(/\s+/).filter(Boolean);
if (skipped.some((word) => known.has(word))) return null;
const match = [found[0], ...found.slice(2)];
const after = rest.slice(match[0].length);
const percent = percentPattern.exec(after);
return {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3]?.toLowerCase(),
percent: percent ? Number(percent[1]) : null,
};
}
const toCatalog = (key, value, unit) =>
key === "energy-kcal"
? value
: convertMass(value, unit === "mcg" ? "µg" : unit, nutrientUnit(key));
/** Resolves an amount, using its %DV to correct misreads such as 8g → "89". */
function resolve(key, reading) {
const dv = dailyValues[key];
let unit = reading.unit || defaultUnits[key] || dv?.[1] || "g";
if (unit === "iu" || unit === "kj" || unit === "kcal") return null;
if (unit === "g" && dv?.[1] === "mcg") unit = "mcg"; // "mcg" read as "g"
let value = reading.less ? reading.value / 2 : reading.value;
if (dv && reading.percent !== null) {
const daily = convertMass(dv[0], mass(dv[1]), mass(unit));
// %DV is rounded (and may read 0 below 2%), so compare in points.
const consistent = (candidate) =>
Math.abs((candidate / daily) * 100 - reading.percent) <=
Math.max(2, reading.percent * 0.3);
if (daily && !consistent(value)) {
const candidates = [];
// A unit read as a digit: "8g" → "89", "160mg" → "16019".
if (!reading.unit && /\d9$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -1)));
if (!reading.unit && /\d19$/.test(reading.raw))
candidates.push(Number(reading.raw.slice(0, -2)));
// A lost decimal point: "05" → 0.5, "25" → 2.5.
if (/^\d{2,3}$/.test(reading.raw))
candidates.push(Number(reading.raw) / 10);
const fixed = candidates.find(consistent);
if (fixed !== undefined) value = fixed;
// Nothing plausible was read; trust the percentage.
else if (!reading.unit) value = (reading.percent / 100) * daily;
}
}
const result = toCatalog(key, value, unit);
// Far beyond a day's worth in one serving is a misread, not a food.
if (dv && result > 10 * toCatalog(key, dv[0], dv[1])) return null;
return result;
}
const mass = (unit) => (unit === "mcg" ? "µg" : unit);
function percentOnly(key, rest) {
const dv = dailyValues[key];
const match = /^[^\d]{0,6}(\d{1,4})\s*%/.exec(rest);
if (!dv || !match) return null;
return toCatalog(key, (Number(match[1]) / 100) * dv[0], dv[1]);
}
/**
* @param {{text: string, x0: number, y0: number, x1: number, y1: number}[]} lines
* @returns {{serving?: string, nutrients: Record<string, number>,
* extra: Record<string, number>, found: string[], text: string[]}}
*/
export function parseLabel(lines) {
const text = rows(lines);
const values = {};
const claimed = new Set(); // character spans already used, per row
const used = (row, start, end) => {
for (let i = start; i < end; i++)
if (claimed.has(`${row}:${i}`)) return true;
for (let i = start; i < end; i++) claimed.add(`${row}:${i}`);
return false;
};
// "Includes 10g Added Sugars" puts the amount before the name.
text.forEach((row, index) => {
const match =
/\bincl(?:udes|\.)?\s*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?\s*(?:of\s*)?added\s*sugars?/i.exec(
row,
) ||
/\b(?:added\s*sugars?|azucares\s*anadidos)[\s:]*(<\s*)?(\d+(?:\.\d+)?)\s*(g)?/i.exec(
row,
);
if (!match || values["added-sugars"] !== undefined) return;
used(index, match.index, match.index + match[0].length);
const percent = percentPattern.exec(
row.slice(match.index + match[0].length),
);
values["added-sugars"] = resolve("added-sugars", {
less: Boolean(match[1]),
raw: match[2],
value: Number(match[2]),
unit: match[3],
percent: percent ? Number(percent[1]) : null,
});
});
for (const [key, pattern] of fields) {
if (values[key] !== undefined) continue;
for (
let index = 0;
index < text.length && values[key] === undefined;
index++
) {
const regex = new RegExp(pattern.source, "gi");
for (let match; (match = regex.exec(text[index])); ) {
const start = match.index;
const end = start + match[0].length;
if (used(index, start, end)) continue;
let rest = text[index].slice(end);
// A linear label can wrap between a name and its amount.
if (
/^[\s,.:]*$/.test(rest) &&
/^<?\s*\.?\d/.test(text[index + 1] || "")
)
rest = " " + text[index + 1];
const reading = readAmount(rest);
const value = reading ? resolve(key, reading) : percentOnly(key, rest);
if (value === null || value === undefined) continue;
values[key] = value;
break;
}
}
}
if (values.salt !== undefined) {
if (values.sodium === undefined) values.sodium = (values.salt / 2.5) * 1000;
delete values.salt;
}
// Calories: the amount follows on the same row, or sits alone on a
// neighbouring row when it's printed in a much larger font.
for (let index = 0; index < text.length; index++) {
const row = text[index];
const match =
/\bcalories\b(?!\s*from)|\benergy\b|\benerg(?:ia|etico)\b/i.exec(row);
if (!match) continue;
const rest = row.slice(match.index + match[0].length);
const kcal =
/(\d+(?:\.\d+)?)\s*kcal/i.exec(rest) ||
/^[^\d]{0,24}?(\d{1,4})(?!\s*(?:k?j|%|m?c?g))\b/i.exec(rest);
const before = text[index - 1] || "";
const after = text[index + 1] || "";
const perServing =
/amount\s*per\s*serving[:\s]*(\d{1,4})\b(?!\s*(?:m?c?g|%))/i;
const alone =
// Big type often lands beside "Amount per serving" on the next row.
perServing.exec(before) ||
perServing.exec(after) ||
// A number alone at the end of a neighbouring row.
[after, before]
.map((neighbour) =>
/%|\d\s*m?c?g\b/i.test(neighbour)
? null
: /(?:^|\s)(\d{1,4})$/.exec(neighbour),
)
.find(Boolean) ||
// A linear label wrapped right after "Calories": "80, Total Fat…".
/^(\d{1,4})\b(?!\s*(?:m?c?g|%|\.\d))/.exec(after);
const value = Number((kcal || alone)?.[1]);
if (Number.isFinite(value) && value <= 5000 && (kcal || alone)) {
values["energy-kcal"] = value;
break;
}
}
let serving;
const perHundred = text.some((row) => /\bper\s*100\s*(g|ml)\b/i.test(row));
for (let index = 0; index < text.length; index++) {
const match = /\bserv(?:ing|\.)\s*size\b[\s:]*(.*)$/i.exec(text[index]);
if (!match) continue;
let size = match[1].trim();
if (!/\d/.test(size)) size = `${size} ${text[index + 1] || ""}`.trim();
// Two-column layouts can run nutrient rows into the serving text.
size = size
.split(/\b(?:calories|amount|total|sodium|protein|servings)\b|%/i)[0]
.trim()
.replace(/\s*\(\s*/g, " (")
.replace(/\s*\)\s*/g, ")")
.replace(/(\d)\s*(g|ml)\b/gi, "$1 $2")
.replace(/\)\s*.*$/, ")")
.trim();
if (size && /\d/.test(size)) serving = size.slice(0, 100);
break;
}
if (!serving && perHundred) {
const ml = text.some((row) => /\bper\s*100\s*ml\b/i.test(row));
serving = ml ? "100 ml" : "100 g";
}
const core = {
"energy-kcal": "calories",
proteins: "protein",
carbohydrates: "carbs",
fat: "fat",
fiber: "fiber",
sugars: "sugar",
sodium: "sodium",
};
const nutrients = {};
const extra = {};
for (const [key, value] of Object.entries(values)) {
if (!Number.isFinite(value) || value < 0 || value > 100000) continue;
const rounded = Math.round(value * 1000) / 1000;
if (core[key]) nutrients[core[key]] = rounded;
else extra[key] = rounded;
}
return {
...(serving ? { serving } : {}),
nutrients,
extra,
found: Object.keys(values),
text,
};
}