All checks were successful
Verify and publish Calio container / build-and-push (push) Successful in 1m2s
Barcode scanning - Scan barcode in the food sheet reads UPC-A/E and EAN-13/8 live from the camera with ZXing WebAssembly (bundled, precached, CSP allows wasm), with photo and typed-digit fallbacks - Lookups check saved foods first (offline), then local copies of USDA FoodData Central Branded Foods and Open Food Facts, then the live Open Food Facts API; API answers and misses are cached - Server imports refresh in a low-priority child process: USDA when a new archive is published, Open Food Facts weekly by streaming its CSV export; each import builds a new table and swaps it in atomically - Every available nutrient is kept: foods and entries gain optional barcode (GTIN-14) and extra nutrients, shown under More nutrients and included in JSON exports - Settings shows the size, date and any error for each product copy Label scanning - Scan label sends a downscaled photo to the server, read with PaddleOCR PP-OCRv5 (server detection, English recognition) on ONNX Runtime CPU, about 1.3 s median - Parser handles US/Canadian, linear, Spanish and EU per-100 g panels, tilted photos, OCR misspellings and %DV cross-checks - Models are pinned and checksum-verified at image build time Container - Runtime moves to Debian slim for ONNX Runtime's glibc build; other platforms' binaries are pruned - Smoke tests now read a fixture label inside the container - README and Unraid docs cover disk use, network access and refresh Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
434 lines
14 KiB
JavaScript
434 lines
14 KiB
JavaScript
// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors
|
|
// PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with
|
|
// unclip, then CTC decoding of each line crop.
|
|
import { readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import ort from "onnxruntime-node";
|
|
import sharp from "sharp";
|
|
|
|
const DET_THRESH = 0.3;
|
|
const BOX_THRESH = 0.6;
|
|
const UNCLIP_RATIO = 1.5;
|
|
const REC_HEIGHT = 48;
|
|
const REC_BATCH = 8;
|
|
|
|
/** Reads the CTC character list out of a PaddleOCR inference.yml. */
|
|
export function readDictionary(yaml) {
|
|
const lines = yaml.split(/\r?\n/);
|
|
const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line));
|
|
if (start < 0) throw new Error("No character_dict in model config");
|
|
const chars = [];
|
|
for (const line of lines.slice(start + 1)) {
|
|
const match = /^\s*- (.*)$/.exec(line);
|
|
if (!match) break;
|
|
let value = match[1];
|
|
if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'");
|
|
else if (value.startsWith('"')) value = JSON.parse(value);
|
|
chars.push(value);
|
|
}
|
|
return ["", ...chars, " "]; // blank, dictionary, then the space character
|
|
}
|
|
|
|
// ---------- geometry ----------
|
|
|
|
function convexHull(points) {
|
|
points.sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
const cross = (o, a, b) =>
|
|
(a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]);
|
|
const lower = [];
|
|
for (const p of points) {
|
|
while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0)
|
|
lower.pop();
|
|
lower.push(p);
|
|
}
|
|
const upper = [];
|
|
for (let i = points.length - 1; i >= 0; i--) {
|
|
const p = points[i];
|
|
while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0)
|
|
upper.pop();
|
|
upper.push(p);
|
|
}
|
|
return lower.slice(0, -1).concat(upper.slice(0, -1));
|
|
}
|
|
|
|
/** Minimum-area rectangle as centre, unit axes and side lengths. */
|
|
export function minAreaRect(points) {
|
|
const hull = convexHull(points);
|
|
if (hull.length < 3) {
|
|
const [x, y] = hull[0] || [0, 0];
|
|
return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 };
|
|
}
|
|
let best = null;
|
|
for (let i = 0; i < hull.length; i++) {
|
|
const [x1, y1] = hull[i];
|
|
const [x2, y2] = hull[(i + 1) % hull.length];
|
|
const length = Math.hypot(x2 - x1, y2 - y1);
|
|
if (!length) continue;
|
|
const ux = (x2 - x1) / length;
|
|
const uy = (y2 - y1) / length;
|
|
let minU = Infinity;
|
|
let maxU = -Infinity;
|
|
let minV = Infinity;
|
|
let maxV = -Infinity;
|
|
for (const [x, y] of hull) {
|
|
const u = x * ux + y * uy;
|
|
const v = -x * uy + y * ux;
|
|
minU = Math.min(minU, u);
|
|
maxU = Math.max(maxU, u);
|
|
minV = Math.min(minV, v);
|
|
maxV = Math.max(maxV, v);
|
|
}
|
|
const area = (maxU - minU) * (maxV - minV);
|
|
if (!best || area < best.area) {
|
|
const cu = (minU + maxU) / 2;
|
|
const cv = (minV + maxV) / 2;
|
|
best = {
|
|
area,
|
|
cx: cu * ux - cv * uy,
|
|
cy: cu * uy + cv * ux,
|
|
ux,
|
|
uy,
|
|
w: maxU - minU,
|
|
h: maxV - minV,
|
|
};
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
/**
|
|
* Orients a rectangle for reading: the axis nearer horizontal runs left to
|
|
* right, the other top to bottom. Returns the top-left corner and axes.
|
|
*/
|
|
function readingFrame({ cx, cy, ux, uy, w, h }) {
|
|
let ax = [ux, uy, w];
|
|
let bx = [-uy, ux, h];
|
|
if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax];
|
|
if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]];
|
|
if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]];
|
|
const [dx, dy, width] = ax;
|
|
const [ex, ey, height] = bx;
|
|
return {
|
|
x: cx - (dx * width) / 2 - (ex * height) / 2,
|
|
y: cy - (dy * width) / 2 - (ey * height) / 2,
|
|
dx,
|
|
dy,
|
|
ex,
|
|
ey,
|
|
width,
|
|
height,
|
|
};
|
|
}
|
|
|
|
// ---------- detection ----------
|
|
|
|
function detectionBoxes(prob, width, height) {
|
|
const visited = new Uint8Array(width * height);
|
|
const boxes = [];
|
|
const stack = [];
|
|
for (let start = 0; start < prob.length; start++) {
|
|
if (visited[start] || prob[start] <= DET_THRESH) continue;
|
|
// Flood-fill one connected region, keeping each row's extreme pixels as
|
|
// hull candidates.
|
|
const rows = new Map();
|
|
let sum = 0;
|
|
let count = 0;
|
|
stack.push(start);
|
|
visited[start] = 1;
|
|
while (stack.length) {
|
|
const index = stack.pop();
|
|
const x = index % width;
|
|
const y = (index - x) / width;
|
|
sum += prob[index];
|
|
count++;
|
|
const row = rows.get(y);
|
|
if (!row) rows.set(y, [x, x]);
|
|
else {
|
|
if (x < row[0]) row[0] = x;
|
|
if (x > row[1]) row[1] = x;
|
|
}
|
|
for (const next of [
|
|
x > 0 ? index - 1 : -1,
|
|
x < width - 1 ? index + 1 : -1,
|
|
y > 0 ? index - width : -1,
|
|
y < height - 1 ? index + width : -1,
|
|
])
|
|
if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) {
|
|
visited[next] = 1;
|
|
stack.push(next);
|
|
}
|
|
}
|
|
if (count < 4 || sum / count < BOX_THRESH) continue;
|
|
const points = [];
|
|
for (const [y, [x0, x1]] of rows)
|
|
points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]);
|
|
const rect = minAreaRect(points);
|
|
if (Math.min(rect.w, rect.h) < 3) continue;
|
|
// Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper.
|
|
const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h));
|
|
rect.w += 2 * d;
|
|
rect.h += 2 * d;
|
|
if (Math.min(rect.w, rect.h) < 5) continue;
|
|
boxes.push({ ...rect, score: sum / count });
|
|
}
|
|
return boxes;
|
|
}
|
|
|
|
// ---------- recognition ----------
|
|
|
|
/** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */
|
|
function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) {
|
|
const { data, width, height } = image;
|
|
const plane = REC_HEIGHT * batchWidth;
|
|
const scale = frame.height / REC_HEIGHT;
|
|
const taps = Math.min(4, Math.max(1, Math.round(scale)));
|
|
const pixel = (x, y, c) => {
|
|
const xi = Math.min(width - 1, Math.max(0, x));
|
|
const yi = Math.min(height - 1, Math.max(0, y));
|
|
return data[(yi * width + xi) * 3 + c];
|
|
};
|
|
for (let ty = 0; ty < REC_HEIGHT; ty++)
|
|
for (let tx = 0; tx < cropWidth; tx++) {
|
|
let r = 0;
|
|
let g = 0;
|
|
let b = 0;
|
|
for (let sy = 0; sy < taps; sy++)
|
|
for (let sx = 0; sx < taps; sx++) {
|
|
const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width;
|
|
const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height;
|
|
const x = frame.x + frame.dx * u + frame.ex * v - 0.5;
|
|
const y = frame.y + frame.dy * u + frame.ey * v - 0.5;
|
|
const x0 = Math.floor(x);
|
|
const y0 = Math.floor(y);
|
|
const fx = x - x0;
|
|
const fy = y - y0;
|
|
for (let c = 0; c < 3; c++) {
|
|
const value =
|
|
pixel(x0, y0, c) * (1 - fx) * (1 - fy) +
|
|
pixel(x0 + 1, y0, c) * fx * (1 - fy) +
|
|
pixel(x0, y0 + 1, c) * (1 - fx) * fy +
|
|
pixel(x0 + 1, y0 + 1, c) * fx * fy;
|
|
if (c === 0) r += value;
|
|
else if (c === 1) g += value;
|
|
else b += value;
|
|
}
|
|
}
|
|
const n = taps * taps * 127.5;
|
|
const at = offset + ty * batchWidth + tx;
|
|
out[at] = b / n - 1;
|
|
out[at + plane] = g / n - 1;
|
|
out[at + 2 * plane] = r / n - 1;
|
|
}
|
|
}
|
|
|
|
function ctcDecode(output, dims, chars) {
|
|
const [count, steps, classes] = dims;
|
|
const results = [];
|
|
for (let n = 0; n < count; n++) {
|
|
let text = "";
|
|
let confidence = 0;
|
|
let kept = 0;
|
|
let previous = -1;
|
|
for (let t = 0; t < steps; t++) {
|
|
const base = (n * steps + t) * classes;
|
|
let best = 0;
|
|
let bestValue = output[base];
|
|
for (let c = 1; c < classes; c++)
|
|
if (output[base + c] > bestValue) {
|
|
best = c;
|
|
bestValue = output[base + c];
|
|
}
|
|
if (best !== 0 && best !== previous) {
|
|
text += chars[best] ?? "";
|
|
confidence += bestValue;
|
|
kept++;
|
|
}
|
|
previous = best;
|
|
}
|
|
results.push({ text, score: kept ? confidence / kept : 0 });
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// ---------- pipeline ----------
|
|
|
|
export async function createOcr(modelDir, { detSide = 1600 } = {}) {
|
|
const options = {
|
|
graphOptimizationLevel: "all",
|
|
// The arena would keep ~2 GB of detection buffers after the first scan.
|
|
enableCpuMemArena: false,
|
|
};
|
|
const [det, rec] = await Promise.all([
|
|
ort.InferenceSession.create(
|
|
join(modelDir, "det", "inference.onnx"),
|
|
options,
|
|
),
|
|
ort.InferenceSession.create(
|
|
join(modelDir, "rec", "inference.onnx"),
|
|
options,
|
|
),
|
|
]);
|
|
const chars = readDictionary(
|
|
readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"),
|
|
);
|
|
|
|
async function detect(image) {
|
|
// Small photos are enlarged too: tiny print detects far better.
|
|
const ratio = Math.min(2, detSide / Math.max(image.width, image.height));
|
|
const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32);
|
|
const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32);
|
|
const { data } = await sharp(image.data, {
|
|
raw: { width: image.width, height: image.height, channels: 3 },
|
|
})
|
|
.resize(w, h, { fit: "fill" })
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
const input = new Float32Array(3 * w * h);
|
|
const mean = [0.406, 0.456, 0.485]; // applied to B, G, R
|
|
const std = [0.225, 0.224, 0.229];
|
|
for (let i = 0; i < w * h; i++)
|
|
for (let c = 0; c < 3; c++)
|
|
// Channel c of the tensor is BGR; data is RGB.
|
|
input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c];
|
|
const result = await det.run({
|
|
[det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]),
|
|
});
|
|
const prob = result[det.outputNames[0]].data;
|
|
const sx = image.width / w;
|
|
const sy = image.height / h;
|
|
return detectionBoxes(prob, w, h).map((box) => {
|
|
// Scale the rectangle back to the full image.
|
|
const ax = box.ux * sx;
|
|
const ay = box.uy * sy;
|
|
const length = Math.hypot(ax, ay);
|
|
const bx = -box.uy * sx;
|
|
const by = box.ux * sy;
|
|
return {
|
|
cx: box.cx * sx,
|
|
cy: box.cy * sy,
|
|
ux: ax / length,
|
|
uy: ay / length,
|
|
w: box.w * length,
|
|
h: box.h * Math.hypot(bx, by),
|
|
score: box.score,
|
|
};
|
|
});
|
|
}
|
|
|
|
async function recognize(image, boxes) {
|
|
const frames = boxes.map((box) => {
|
|
const frame = readingFrame(box);
|
|
return { frame, ratio: frame.width / Math.max(1, frame.height) };
|
|
});
|
|
const order = frames
|
|
.map((_, i) => i)
|
|
.sort((a, b) => frames[a].ratio - frames[b].ratio);
|
|
const texts = new Array(frames.length);
|
|
for (let start = 0; start < order.length; start += REC_BATCH) {
|
|
const batch = order.slice(start, start + REC_BATCH);
|
|
const maxRatio = Math.max(
|
|
320 / REC_HEIGHT,
|
|
...batch.map((i) => frames[i].ratio),
|
|
);
|
|
const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio));
|
|
const plane = REC_HEIGHT * batchWidth;
|
|
const input = new Float32Array(batch.length * 3 * plane);
|
|
batch.forEach((i, n) => {
|
|
const cropWidth = Math.min(
|
|
batchWidth,
|
|
Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)),
|
|
);
|
|
sampleCrop(
|
|
image,
|
|
frames[i].frame,
|
|
input,
|
|
n * 3 * plane,
|
|
batchWidth,
|
|
cropWidth,
|
|
);
|
|
});
|
|
const result = await rec.run({
|
|
[rec.inputNames[0]]: new ort.Tensor("float32", input, [
|
|
batch.length,
|
|
3,
|
|
REC_HEIGHT,
|
|
batchWidth,
|
|
]),
|
|
});
|
|
const output = result[rec.outputNames[0]];
|
|
ctcDecode(output.data, output.dims, chars).forEach((value, n) => {
|
|
texts[batch[n]] = value;
|
|
});
|
|
}
|
|
return boxes.map((box, i) => {
|
|
const { frame } = frames[i];
|
|
const xs = [
|
|
frame.x,
|
|
frame.x + frame.dx * frame.width,
|
|
frame.x + frame.ex * frame.height,
|
|
frame.x + frame.dx * frame.width + frame.ex * frame.height,
|
|
];
|
|
const ys = [
|
|
frame.y,
|
|
frame.y + frame.dy * frame.width,
|
|
frame.y + frame.ey * frame.height,
|
|
frame.y + frame.dy * frame.width + frame.ey * frame.height,
|
|
];
|
|
return {
|
|
text: texts[i].text,
|
|
score: texts[i].score,
|
|
angle: Math.atan2(frame.dy, frame.dx),
|
|
cx: (Math.min(...xs) + Math.max(...xs)) / 2,
|
|
cy: (Math.min(...ys) + Math.max(...ys)) / 2,
|
|
x0: Math.min(...xs),
|
|
y0: Math.min(...ys),
|
|
x1: Math.max(...xs),
|
|
y1: Math.max(...ys),
|
|
height: frame.height,
|
|
};
|
|
});
|
|
}
|
|
|
|
async function decode(buffer, rotate = 0) {
|
|
const { data, info } = await sharp(buffer, { failOn: "none" })
|
|
.rotate() // honour EXIF orientation
|
|
.rotate(rotate)
|
|
.resize(2400, 2400, { fit: "inside", withoutEnlargement: true })
|
|
.removeAlpha()
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
return { data, width: info.width, height: info.height };
|
|
}
|
|
|
|
async function readImage(image) {
|
|
const boxes = await detect(image);
|
|
const lines = (await recognize(image, boxes)).filter((line) =>
|
|
line.text.trim(),
|
|
);
|
|
const score = lines.reduce(
|
|
(sum, line) => sum + line.score * line.text.length,
|
|
0,
|
|
);
|
|
return { lines, score, boxes };
|
|
}
|
|
|
|
/** Returns recognised lines with their bounding boxes, in image pixels. */
|
|
async function read(buffer) {
|
|
const image = await decode(buffer);
|
|
let best = await readImage(image);
|
|
// A label photographed sideways yields mostly tall, narrow boxes; try
|
|
// both quarter turns and keep whichever reads more confidently.
|
|
const tall = best.boxes.filter((box) => {
|
|
const frame = readingFrame(box);
|
|
return frame.height > frame.width * 1.5;
|
|
}).length;
|
|
if (best.boxes.length && tall > best.boxes.length / 2)
|
|
for (const angle of [90, 270]) {
|
|
const attempt = await readImage(await decode(buffer, angle));
|
|
if (attempt.score > best.score) best = attempt;
|
|
}
|
|
return best.lines;
|
|
}
|
|
|
|
return { read };
|
|
}
|