// PP-OCRv5 text detection + recognition on ONNX Runtime (CPU). Mirrors // PaddleOCR's own pre- and post-processing: DB detection, min-area boxes with // unclip, then CTC decoding of each line crop. import { readFileSync } from "node:fs"; import { join } from "node:path"; import ort from "onnxruntime-node"; import sharp from "sharp"; const DET_THRESH = 0.3; const BOX_THRESH = 0.6; const UNCLIP_RATIO = 1.5; const REC_HEIGHT = 48; const REC_BATCH = 8; /** Reads the CTC character list out of a PaddleOCR inference.yml. */ export function readDictionary(yaml) { const lines = yaml.split(/\r?\n/); const start = lines.findIndex((line) => /^\s*character_dict:\s*$/.test(line)); if (start < 0) throw new Error("No character_dict in model config"); const chars = []; for (const line of lines.slice(start + 1)) { const match = /^\s*- (.*)$/.exec(line); if (!match) break; let value = match[1]; if (value.startsWith("'")) value = value.slice(1, -1).replace(/''/g, "'"); else if (value.startsWith('"')) value = JSON.parse(value); chars.push(value); } return ["", ...chars, " "]; // blank, dictionary, then the space character } // ---------- geometry ---------- function convexHull(points) { points.sort((a, b) => a[0] - b[0] || a[1] - b[1]); const cross = (o, a, b) => (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]); const lower = []; for (const p of points) { while (lower.length >= 2 && cross(lower.at(-2), lower.at(-1), p) <= 0) lower.pop(); lower.push(p); } const upper = []; for (let i = points.length - 1; i >= 0; i--) { const p = points[i]; while (upper.length >= 2 && cross(upper.at(-2), upper.at(-1), p) <= 0) upper.pop(); upper.push(p); } return lower.slice(0, -1).concat(upper.slice(0, -1)); } /** Minimum-area rectangle as centre, unit axes and side lengths. */ export function minAreaRect(points) { const hull = convexHull(points); if (hull.length < 3) { const [x, y] = hull[0] || [0, 0]; return { cx: x, cy: y, ux: 1, uy: 0, w: 1, h: 1 }; } let best = null; for (let i = 0; i < hull.length; i++) { const [x1, y1] = hull[i]; const [x2, y2] = hull[(i + 1) % hull.length]; const length = Math.hypot(x2 - x1, y2 - y1); if (!length) continue; const ux = (x2 - x1) / length; const uy = (y2 - y1) / length; let minU = Infinity; let maxU = -Infinity; let minV = Infinity; let maxV = -Infinity; for (const [x, y] of hull) { const u = x * ux + y * uy; const v = -x * uy + y * ux; minU = Math.min(minU, u); maxU = Math.max(maxU, u); minV = Math.min(minV, v); maxV = Math.max(maxV, v); } const area = (maxU - minU) * (maxV - minV); if (!best || area < best.area) { const cu = (minU + maxU) / 2; const cv = (minV + maxV) / 2; best = { area, cx: cu * ux - cv * uy, cy: cu * uy + cv * ux, ux, uy, w: maxU - minU, h: maxV - minV, }; } } return best; } /** * Orients a rectangle for reading: the axis nearer horizontal runs left to * right, the other top to bottom. Returns the top-left corner and axes. */ function readingFrame({ cx, cy, ux, uy, w, h }) { let ax = [ux, uy, w]; let bx = [-uy, ux, h]; if (Math.abs(bx[0]) > Math.abs(ax[0])) [ax, bx] = [bx, ax]; if (ax[0] < 0) ax = [-ax[0], -ax[1], ax[2]]; if (bx[1] < 0) bx = [-bx[0], -bx[1], bx[2]]; const [dx, dy, width] = ax; const [ex, ey, height] = bx; return { x: cx - (dx * width) / 2 - (ex * height) / 2, y: cy - (dy * width) / 2 - (ey * height) / 2, dx, dy, ex, ey, width, height, }; } // ---------- detection ---------- function detectionBoxes(prob, width, height) { const visited = new Uint8Array(width * height); const boxes = []; const stack = []; for (let start = 0; start < prob.length; start++) { if (visited[start] || prob[start] <= DET_THRESH) continue; // Flood-fill one connected region, keeping each row's extreme pixels as // hull candidates. const rows = new Map(); let sum = 0; let count = 0; stack.push(start); visited[start] = 1; while (stack.length) { const index = stack.pop(); const x = index % width; const y = (index - x) / width; sum += prob[index]; count++; const row = rows.get(y); if (!row) rows.set(y, [x, x]); else { if (x < row[0]) row[0] = x; if (x > row[1]) row[1] = x; } for (const next of [ x > 0 ? index - 1 : -1, x < width - 1 ? index + 1 : -1, y > 0 ? index - width : -1, y < height - 1 ? index + width : -1, ]) if (next >= 0 && !visited[next] && prob[next] > DET_THRESH) { visited[next] = 1; stack.push(next); } } if (count < 4 || sum / count < BOX_THRESH) continue; const points = []; for (const [y, [x0, x1]] of rows) points.push([x0, y], [x1 + 1, y], [x0, y + 1], [x1 + 1, y + 1]); const rect = minAreaRect(points); if (Math.min(rect.w, rect.h) < 3) continue; // Expand by the DB "unclip" distance, as PaddleOCR does with pyclipper. const d = (rect.w * rect.h * UNCLIP_RATIO) / (2 * (rect.w + rect.h)); rect.w += 2 * d; rect.h += 2 * d; if (Math.min(rect.w, rect.h) < 5) continue; boxes.push({ ...rect, score: sum / count }); } return boxes; } // ---------- recognition ---------- /** Samples a rectangle of the RGB image into a normalised BGR tensor slot. */ function sampleCrop(image, frame, out, offset, batchWidth, cropWidth) { const { data, width, height } = image; const plane = REC_HEIGHT * batchWidth; const scale = frame.height / REC_HEIGHT; const taps = Math.min(4, Math.max(1, Math.round(scale))); const pixel = (x, y, c) => { const xi = Math.min(width - 1, Math.max(0, x)); const yi = Math.min(height - 1, Math.max(0, y)); return data[(yi * width + xi) * 3 + c]; }; for (let ty = 0; ty < REC_HEIGHT; ty++) for (let tx = 0; tx < cropWidth; tx++) { let r = 0; let g = 0; let b = 0; for (let sy = 0; sy < taps; sy++) for (let sx = 0; sx < taps; sx++) { const u = ((tx + (sx + 0.5) / taps) / cropWidth) * frame.width; const v = ((ty + (sy + 0.5) / taps) / REC_HEIGHT) * frame.height; const x = frame.x + frame.dx * u + frame.ex * v - 0.5; const y = frame.y + frame.dy * u + frame.ey * v - 0.5; const x0 = Math.floor(x); const y0 = Math.floor(y); const fx = x - x0; const fy = y - y0; for (let c = 0; c < 3; c++) { const value = pixel(x0, y0, c) * (1 - fx) * (1 - fy) + pixel(x0 + 1, y0, c) * fx * (1 - fy) + pixel(x0, y0 + 1, c) * (1 - fx) * fy + pixel(x0 + 1, y0 + 1, c) * fx * fy; if (c === 0) r += value; else if (c === 1) g += value; else b += value; } } const n = taps * taps * 127.5; const at = offset + ty * batchWidth + tx; out[at] = b / n - 1; out[at + plane] = g / n - 1; out[at + 2 * plane] = r / n - 1; } } function ctcDecode(output, dims, chars) { const [count, steps, classes] = dims; const results = []; for (let n = 0; n < count; n++) { let text = ""; let confidence = 0; let kept = 0; let previous = -1; for (let t = 0; t < steps; t++) { const base = (n * steps + t) * classes; let best = 0; let bestValue = output[base]; for (let c = 1; c < classes; c++) if (output[base + c] > bestValue) { best = c; bestValue = output[base + c]; } if (best !== 0 && best !== previous) { text += chars[best] ?? ""; confidence += bestValue; kept++; } previous = best; } results.push({ text, score: kept ? confidence / kept : 0 }); } return results; } // ---------- pipeline ---------- export async function createOcr(modelDir, { detSide = 1600 } = {}) { const options = { graphOptimizationLevel: "all", // The arena would keep ~2 GB of detection buffers after the first scan. enableCpuMemArena: false, }; const [det, rec] = await Promise.all([ ort.InferenceSession.create( join(modelDir, "det", "inference.onnx"), options, ), ort.InferenceSession.create( join(modelDir, "rec", "inference.onnx"), options, ), ]); const chars = readDictionary( readFileSync(join(modelDir, "rec", "inference.yml"), "utf8"), ); async function detect(image) { // Small photos are enlarged too: tiny print detects far better. const ratio = Math.min(2, detSide / Math.max(image.width, image.height)); const w = Math.max(32, Math.round((image.width * ratio) / 32) * 32); const h = Math.max(32, Math.round((image.height * ratio) / 32) * 32); const { data } = await sharp(image.data, { raw: { width: image.width, height: image.height, channels: 3 }, }) .resize(w, h, { fit: "fill" }) .raw() .toBuffer({ resolveWithObject: true }); const input = new Float32Array(3 * w * h); const mean = [0.406, 0.456, 0.485]; // applied to B, G, R const std = [0.225, 0.224, 0.229]; for (let i = 0; i < w * h; i++) for (let c = 0; c < 3; c++) // Channel c of the tensor is BGR; data is RGB. input[c * w * h + i] = (data[i * 3 + (2 - c)] / 255 - mean[c]) / std[c]; const result = await det.run({ [det.inputNames[0]]: new ort.Tensor("float32", input, [1, 3, h, w]), }); const prob = result[det.outputNames[0]].data; const sx = image.width / w; const sy = image.height / h; return detectionBoxes(prob, w, h).map((box) => { // Scale the rectangle back to the full image. const ax = box.ux * sx; const ay = box.uy * sy; const length = Math.hypot(ax, ay); const bx = -box.uy * sx; const by = box.ux * sy; return { cx: box.cx * sx, cy: box.cy * sy, ux: ax / length, uy: ay / length, w: box.w * length, h: box.h * Math.hypot(bx, by), score: box.score, }; }); } async function recognize(image, boxes) { const frames = boxes.map((box) => { const frame = readingFrame(box); return { frame, ratio: frame.width / Math.max(1, frame.height) }; }); const order = frames .map((_, i) => i) .sort((a, b) => frames[a].ratio - frames[b].ratio); const texts = new Array(frames.length); for (let start = 0; start < order.length; start += REC_BATCH) { const batch = order.slice(start, start + REC_BATCH); const maxRatio = Math.max( 320 / REC_HEIGHT, ...batch.map((i) => frames[i].ratio), ); const batchWidth = Math.min(3200, Math.ceil(REC_HEIGHT * maxRatio)); const plane = REC_HEIGHT * batchWidth; const input = new Float32Array(batch.length * 3 * plane); batch.forEach((i, n) => { const cropWidth = Math.min( batchWidth, Math.max(1, Math.ceil(REC_HEIGHT * frames[i].ratio)), ); sampleCrop( image, frames[i].frame, input, n * 3 * plane, batchWidth, cropWidth, ); }); const result = await rec.run({ [rec.inputNames[0]]: new ort.Tensor("float32", input, [ batch.length, 3, REC_HEIGHT, batchWidth, ]), }); const output = result[rec.outputNames[0]]; ctcDecode(output.data, output.dims, chars).forEach((value, n) => { texts[batch[n]] = value; }); } return boxes.map((box, i) => { const { frame } = frames[i]; const xs = [ frame.x, frame.x + frame.dx * frame.width, frame.x + frame.ex * frame.height, frame.x + frame.dx * frame.width + frame.ex * frame.height, ]; const ys = [ frame.y, frame.y + frame.dy * frame.width, frame.y + frame.ey * frame.height, frame.y + frame.dy * frame.width + frame.ey * frame.height, ]; return { text: texts[i].text, score: texts[i].score, angle: Math.atan2(frame.dy, frame.dx), cx: (Math.min(...xs) + Math.max(...xs)) / 2, cy: (Math.min(...ys) + Math.max(...ys)) / 2, x0: Math.min(...xs), y0: Math.min(...ys), x1: Math.max(...xs), y1: Math.max(...ys), height: frame.height, }; }); } async function decode(buffer, rotate = 0) { const { data, info } = await sharp(buffer, { failOn: "none" }) .rotate() // honour EXIF orientation .rotate(rotate) .resize(2400, 2400, { fit: "inside", withoutEnlargement: true }) .removeAlpha() .raw() .toBuffer({ resolveWithObject: true }); return { data, width: info.width, height: info.height }; } async function readImage(image) { const boxes = await detect(image); const lines = (await recognize(image, boxes)).filter((line) => line.text.trim(), ); const score = lines.reduce( (sum, line) => sum + line.score * line.text.length, 0, ); return { lines, score, boxes }; } /** Returns recognised lines with their bounding boxes, in image pixels. */ async function read(buffer) { const image = await decode(buffer); let best = await readImage(image); // A label photographed sideways yields mostly tall, narrow boxes; try // both quarter turns and keep whichever reads more confidently. const tall = best.boxes.filter((box) => { const frame = readingFrame(box); return frame.height > frame.width * 1.5; }).length; if (best.boxes.length && tall > best.boxes.length / 2) for (const angle of [90, 270]) { const attempt = await readImage(await decode(buffer, angle)); if (attempt.score > best.score) best = attempt; } return best.lines; } return { read }; }