Add barcode and nutrition label scanning
All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s

Barcode scanning
- Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from
  the camera with ZXing WebAssembly (bundled, precached, CSP allows
  wasm), with photo and typed-digit fallbacks
- Lookups check saved foods first (offline), then local copies of USDA
  FoodData Central Branded Foods and Open Food Facts, then the live
  Open Food Facts API; API answers and misses are cached
- Server imports refresh in a low-priority child process: USDA when a
  new archive is published, Open Food Facts weekly by streaming its CSV
  export; each import builds a new table and swaps it in atomically
- Every available nutrient is kept: foods and entries gain optional
  barcode (GTIN-14) and extra nutrients, shown under More nutrients and
  included in JSON exports
- Settings shows the size, date and any error for each product copy

Label scanning
- Scan label sends a downscaled photo to the server, read with
  PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX
  Runtime CPU, about 1.3 s median
- Parser handles US/Canadian, linear, Spanish and EU per-100 g panels,
  tilted photos, OCR misspellings and %DV cross-checks
- Models are pinned and checksum-verified at image build time

Container
- Runtime moves to Debian slim for ONNX Runtime's glibc build; other
  platforms' binaries are pruned
- Smoke tests now read a fixture label inside the container
- README and Unraid docs cover disk use, network access and refresh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Elijah 2026-10-22 13:41:44 -07:00
parent 8f6981c8a2
commit 80fc48b9f1
39 changed files with 4434 additions and 33 deletions

434
server/ocr/engine.js Normal file
View file

@ -0,0 +1,434 @@
// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors
// PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with
// unclip, then CTC decoding of each line crop.
import { readFileSync } from "node:fs";
import { join } from "node:path";
import ort from "onnxruntime-node";
import sharp from "sharp";
const DET_THRESH = 0.3;
const BOX_THRESH = 0.6;
const UNCLIP_RATIO = 1.5;
const REC_HEIGHT = 48;
const REC_BATCH = 8;
/** Reads the CTC character list out of a PaddleOCR inference.yml. */
export function readDictionary(yaml) {
const lines = yaml.split(/\r?\n/);
const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line));
if (start < 0) throw new Error("No character_dict in model config");
const chars = [];
for (const line of lines.slice(start + 1)) {
const match = /^\s*- (.*)$/.exec(line);
if (!match) break;
let value = match[1];
if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'");
else if (value.startsWith('"')) value = JSON.parse(value);
chars.push(value);
}
return ["", ...chars, " "]; // blank, dictionary, then the space character
}
// ---------- geometry ----------
function convexHull(points) {
points.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
const cross = (o, a, b) =>
(a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]);
const lower = [];
for (const p of points) {
while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0)
lower.pop();
lower.push(p);
}
const upper = [];
for (let i = points.length - 1; i >= 0; i--) {
const p = points[i];
while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0)
upper.pop();
upper.push(p);
}
return lower.slice(0, -1).concat(upper.slice(0, -1));
}
/** Minimum-area rectangle as centre, unit axes and side lengths. */
export function minAreaRect(points) {
const hull = convexHull(points);
if (hull.length < 3) {
const [x, y] = hull[0] || [0, 0];
return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 };
}
let best = null;
for (let i = 0; i < hull.length; i++) {
const [x1, y1] = hull[i];
const [x2, y2] = hull[(i + 1) % hull.length];
const length = Math.hypot(x2 - x1, y2 - y1);
if (!length) continue;
const ux = (x2 - x1) / length;
const uy = (y2 - y1) / length;
let minU = Infinity;
let maxU = -Infinity;
let minV = Infinity;
let maxV = -Infinity;
for (const [x, y] of hull) {
const u = x * ux + y * uy;
const v = -x * uy + y * ux;
minU = Math.min(minU, u);
maxU = Math.max(maxU, u);
minV = Math.min(minV, v);
maxV = Math.max(maxV, v);
}
const area = (maxU - minU) * (maxV - minV);
if (!best || area < best.area) {
const cu = (minU + maxU) / 2;
const cv = (minV + maxV) / 2;
best = {
area,
cx: cu * ux - cv * uy,
cy: cu * uy + cv * ux,
ux,
uy,
w: maxU - minU,
h: maxV - minV,
};
}
}
return best;
}
/**
* Orients a rectangle for reading: the axis nearer horizontal runs left to
* right, the other top to bottom. Returns the top-left corner and axes.
*/
function readingFrame({ cx, cy, ux, uy, w, h }) {
let ax = [ux, uy, w];
let bx = [-uy, ux, h];
if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax];
if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]];
if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]];
const [dx, dy, width] = ax;
const [ex, ey, height] = bx;
return {
x: cx - (dx * width) / 2 - (ex * height) / 2,
y: cy - (dy * width) / 2 - (ey * height) / 2,
dx,
dy,
ex,
ey,
width,
height,
};
}
// ---------- detection ----------
function detectionBoxes(prob, width, height) {
const visited = new Uint8Array(width * height);
const boxes = [];
const stack = [];
for (let start = 0; start < prob.length; start++) {
if (visited[start] || prob[start] <= DET_THRESH) continue;
// Flood-fill one connected region, keeping each row's extreme pixels as
// hull candidates.
const rows = new Map();
let sum = 0;
let count = 0;
stack.push(start);
visited[start] = 1;
while (stack.length) {
const index = stack.pop();
const x = index % width;
const y = (index - x) / width;
sum += prob[index];
count++;
const row = rows.get(y);
if (!row) rows.set(y, [x, x]);
else {
if (x < row[0]) row[0] = x;
if (x > row[1]) row[1] = x;
}
for (const next of [
x > 0 ? index - 1 : -1,
x < width - 1 ? index + 1 : -1,
y > 0 ? index - width : -1,
y < height - 1 ? index + width : -1,
])
if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) {
visited[next] = 1;
stack.push(next);
}
}
if (count < 4 || sum / count < BOX_THRESH) continue;
const points = [];
for (const [y, [x0, x1]] of rows)
points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]);
const rect = minAreaRect(points);
if (Math.min(rect.w, rect.h) < 3) continue;
// Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper.
const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h));
rect.w += 2 * d;
rect.h += 2 * d;
if (Math.min(rect.w, rect.h) < 5) continue;
boxes.push({ ...rect, score: sum / count });
}
return boxes;
}
// ---------- recognition ----------
/** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */
function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) {
const { data, width, height } = image;
const plane = REC_HEIGHT * batchWidth;
const scale = frame.height / REC_HEIGHT;
const taps = Math.min(4, Math.max(1, Math.round(scale)));
const pixel = (x, y, c) => {
const xi = Math.min(width - 1, Math.max(0, x));
const yi = Math.min(height - 1, Math.max(0, y));
return data[(yi * width + xi) * 3 + c];
};
for (let ty = 0; ty < REC_HEIGHT; ty++)
for (let tx = 0; tx < cropWidth; tx++) {
let r = 0;
let g = 0;
let b = 0;
for (let sy = 0; sy < taps; sy++)
for (let sx = 0; sx < taps; sx++) {
const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width;
const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height;
const x = frame.x + frame.dx * u + frame.ex * v - 0.5;
const y = frame.y + frame.dy * u + frame.ey * v - 0.5;
const x0 = Math.floor(x);
const y0 = Math.floor(y);
const fx = x - x0;
const fy = y - y0;
for (let c = 0; c < 3; c++) {
const value =
pixel(x0, y0, c) * (1 - fx) * (1 - fy) +
pixel(x0 + 1, y0, c) * fx * (1 - fy) +
pixel(x0, y0 + 1, c) * (1 - fx) * fy +
pixel(x0 + 1, y0 + 1, c) * fx * fy;
if (c === 0) r += value;
else if (c === 1) g += value;
else b += value;
}
}
const n = taps * taps * 127.5;
const at = offset + ty * batchWidth + tx;
out[at] = b / n - 1;
out[at + plane] = g / n - 1;
out[at + 2 * plane] = r / n - 1;
}
}
function ctcDecode(output, dims, chars) {
const [count, steps, classes] = dims;
const results = [];
for (let n = 0; n < count; n++) {
let text = "";
let confidence = 0;
let kept = 0;
let previous = -1;
for (let t = 0; t < steps; t++) {
const base = (n * steps + t) * classes;
let best = 0;
let bestValue = output[base];
for (let c = 1; c < classes; c++)
if (output[base + c] > bestValue) {
best = c;
bestValue = output[base + c];
}
if (best !== 0 && best !== previous) {
text += chars[best] ?? "";
confidence += bestValue;
kept++;
}
previous = best;
}
results.push({ text, score: kept ? confidence / kept : 0 });
}
return results;
}
// ---------- pipeline ----------
export async function createOcr(modelDir, { detSide = 1600 } = {}) {
const options = {
graphOptimizationLevel: "all",
// The arena would keep ~2 GB of detection buffers after the first scan.
enableCpuMemArena: false,
};
const [det, rec] = await Promise.all([
ort.InferenceSession.create(
join(modelDir, "det", "inference.onnx"),
options,
),
ort.InferenceSession.create(
join(modelDir, "rec", "inference.onnx"),
options,
),
]);
const chars = readDictionary(
readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"),
);
async function detect(image) {
// Small photos are enlarged too: tiny print detects far better.
const ratio = Math.min(2, detSide / Math.max(image.width, image.height));
const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32);
const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
const { data } = await sharp(image.data, {
raw: { width: image.width, height: image.height, channels: 3 },
})
.resize(w, h, { fit: "fill" })
.raw()
.toBuffer({ resolveWithObject: true });
const input = new Float32Array(3 * w * h);
const mean = [0.406, 0.456, 0.485]; // applied to B, G, R
const std = [0.225, 0.224, 0.229];
for (let i = 0; i < w * h; i++)
for (let c = 0; c < 3; c++)
// Channel c of the tensor is BGR; data is RGB.
input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c];
const result = await det.run({
[det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]),
});
const prob = result[det.outputNames[0]].data;
const sx = image.width / w;
const sy = image.height / h;
return detectionBoxes(prob, w, h).map((box) => {
// Scale the rectangle back to the full image.
const ax = box.ux * sx;
const ay = box.uy * sy;
const length = Math.hypot(ax, ay);
const bx = -box.uy * sx;
const by = box.ux * sy;
return {
cx: box.cx * sx,
cy: box.cy * sy,
ux: ax / length,
uy: ay / length,
w: box.w * length,
h: box.h * Math.hypot(bx, by),
score: box.score,
};
});
}
async function recognize(image, boxes) {
const frames = boxes.map((box) => {
const frame = readingFrame(box);
return { frame, ratio: frame.width / Math.max(1, frame.height) };
});
const order = frames
.map((_, i) => i)
.sort((a, b) => frames[a].ratio - frames[b].ratio);
const texts = new Array(frames.length);
for (let start = 0; start < order.length; start += REC_BATCH) {
const batch = order.slice(start, start + REC_BATCH);
const maxRatio = Math.max(
320 / REC_HEIGHT,
...batch.map((i) => frames[i].ratio),
);
const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio));
const plane = REC_HEIGHT * batchWidth;
const input = new Float32Array(batch.length * 3 * plane);
batch.forEach((i, n) => {
const cropWidth = Math.min(
batchWidth,
Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)),
);
sampleCrop(
image,
frames[i].frame,
input,
n * 3 * plane,
batchWidth,
cropWidth,
);
});
const result = await rec.run({
[rec.inputNames[0]]: new ort.Tensor("float32", input, [
batch.length,
3,
REC_HEIGHT,
batchWidth,
]),
});
const output = result[rec.outputNames[0]];
ctcDecode(output.data, output.dims, chars).forEach((value, n) => {
texts[batch[n]] = value;
});
}
return boxes.map((box, i) => {
const { frame } = frames[i];
const xs = [
frame.x,
frame.x + frame.dx * frame.width,
frame.x + frame.ex * frame.height,
frame.x + frame.dx * frame.width + frame.ex * frame.height,
];
const ys = [
frame.y,
frame.y + frame.dy * frame.width,
frame.y + frame.ey * frame.height,
frame.y + frame.dy * frame.width + frame.ey * frame.height,
];
return {
text: texts[i].text,
score: texts[i].score,
angle: Math.atan2(frame.dy, frame.dx),
cx: (Math.min(...xs) + Math.max(...xs)) / 2,
cy: (Math.min(...ys) + Math.max(...ys)) / 2,
x0: Math.min(...xs),
y0: Math.min(...ys),
x1: Math.max(...xs),
y1: Math.max(...ys),
height: frame.height,
};
});
}
async function decode(buffer, rotate = 0) {
const { data, info } = await sharp(buffer, { failOn: "none" })
.rotate() // honour EXIF orientation
.rotate(rotate)
.resize(2400, 2400, { fit: "inside", withoutEnlargement: true })
.removeAlpha()
.raw()
.toBuffer({ resolveWithObject: true });
return { data, width: info.width, height: info.height };
}
async function readImage(image) {
const boxes = await detect(image);
const lines = (await recognize(image, boxes)).filter((line) =>
line.text.trim(),
);
const score = lines.reduce(
(sum, line) => sum + line.score * line.text.length,
0,
);
return { lines, score, boxes };
}
/** Returns recognised lines with their bounding boxes, in image pixels. */
async function read(buffer) {
const image = await decode(buffer);
let best = await readImage(image);
// A label photographed sideways yields mostly tall, narrow boxes; try
// both quarter turns and keep whichever reads more confidently.
const tall = best.boxes.filter((box) => {
const frame = readingFrame(box);
return frame.height > frame.width * 1.5;
}).length;
if (best.boxes.length && tall > best.boxes.length / 2)
for (const angle of [90, 270]) {
const attempt = await readImage(await decode(buffer, angle));
if (attempt.score > best.score) best = attempt;
}
return best.lines;
}
return { read };
}