feat: add Dockerfile for ConvertX-CN v0.1.11 with pre-downloaded models and offline-first design
- Introduced a new Dockerfile that builds the ConvertX-CN image with all necessary dependencies and models pre-downloaded. - Implemented multi-stage builds to optimize image size and build time. - Added a script for verifying the integrity of pre-downloaded models and dependencies. - Ensured the image operates in an offline-first manner, eliminating runtime dependencies on external networks.
This commit is contained in:
Your Name
parent
8820aecafe
commit
a9867ee97c
6 changed files with 1284 additions and 270 deletions
388
Dockerfile
388
Dockerfile
|
|
@ -16,7 +16,7 @@
|
|||
#
|
||||
# 🤖 預下載模型清單：
|
||||
# - PDFMathTranslate: DocLayout-YOLO ONNX（佈局分析）
|
||||
# - BabelDOC: 完整資源包（透過 --warmup）
|
||||
# - BabelDOC: DocLayout-YOLO + 字型資源（顯式下載，無 warmup）
|
||||
# - MinerU: PDF-Extract-Kit-1.0（Pipeline 模型）
|
||||
# 包含：DocLayout-YOLO, YOLOv8 MFD, UniMERNet, PaddleOCR, LayoutReader, SLANet
|
||||
#
|
||||
|
|
@ -26,6 +26,12 @@
|
|||
# - 確保 Multi-Arch (amd64/arm64) 構建穩定性
|
||||
# - 避免 trixie (testing) 套件同步不穩定問題
|
||||
#
|
||||
# 🔒 Offline-first 設計原則：
|
||||
# - 所有下載行為僅發生在 Docker build 階段
|
||||
# - Runtime 完全離線運行，不依賴任何網路請求
|
||||
# - 禁止任何 CLI warmup / 隱性下載行為
|
||||
# - 所有 cache 在同一 RUN 內清除，避免 layer diff 膨脹
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
FROM debian:bookworm-slim AS base
|
||||
|
|
@ -231,169 +237,99 @@ RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
|||
pipx \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 12：安裝 Python 工具（pipx）+ huggingface_hub（用於模型下載）
|
||||
# 注意：Debian bookworm 使用 PEP 668，需要 --break-system-packages 來安裝系統級套件
|
||||
# 注意：markitdown[all] 可能依賴 transformers 或其他 HuggingFace 套件，需清理 cache
|
||||
RUN pipx install "markitdown[all]" \
|
||||
&& pip3 install --no-cache-dir --break-system-packages huggingface_hub \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /tmp/*
|
||||
|
||||
# 階段 12-A：安裝 pdf2zh（PDFMathTranslate 引擎）
|
||||
# 注意：pipx install 可能觸發依賴套件的隱式下載，結尾必須清理 cache
|
||||
RUN pipx install "pdf2zh" \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /tmp/*
|
||||
|
||||
# 階段 12-B：安裝 babeldoc（BabelDOC 引擎）
|
||||
# BabelDOC 是一個 PDF 翻譯工具，與 pdf2zh 類似但使用不同的翻譯方式
|
||||
# 注意：babeldoc 依賴 transformers，安裝時可能觸發模型 cache
|
||||
RUN (pipx install "babeldoc" || echo "⚠️ babeldoc 安裝失敗，跳過...") \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /tmp/*
|
||||
|
||||
# 階段 13：安裝 mineru（可能在 arm64 上有問題，加入錯誤處理）
|
||||
# 🔴 關鍵：mineru[all] 依賴大量 HuggingFace 套件，安裝過程可能觸發模型下載
|
||||
# 必須在同一 RUN 內清理 cache，否則會在 layer diff 中產生數 GB 的重複資料
|
||||
RUN (pipx install "mineru[all]" || echo "⚠️ mineru 安裝失敗（可能是 arm64 相容性問題），跳過...") \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /root/.cache/torch /tmp/*
|
||||
|
||||
# 最終清理（延後到模型下載完成後）
|
||||
|
||||
# Add pipx bin directory to PATH（必須在模型下載前設定）
|
||||
# ==============================================================================
|
||||
# 🔥 階段 12-UNIFIED：Python 工具安裝 + 模型下載（單一 RUN 原則）
|
||||
# ==============================================================================
|
||||
#
|
||||
# ⚠️ 關鍵設計原則：
|
||||
# 1. 所有 pipx install 和模型下載必須在同一個 RUN 中完成
|
||||
# 2. 所有 cache 在同一個 RUN 結尾清除
|
||||
# 3. 禁止任何 CLI warmup / 隱性下載行為
|
||||
# 4. 僅使用顯式 HuggingFace snapshot_download 下載模型
|
||||
#
|
||||
# ⬇️ 此 RUN 包含所有 Docker build 階段下載：
|
||||
# - Python 工具：markitdown, pdf2zh, babeldoc, mineru
|
||||
# - 模型：DocLayout-YOLO ONNX, MinerU Pipeline 模型
|
||||
# - 字型：GoNotoKurrent, Source Han Serif
|
||||
# - Runtime 不會再下載任何資源
|
||||
#
|
||||
# ==============================================================================
|
||||
ENV PATH="/root/.local/bin:${PATH}"
|
||||
ENV PIPX_HOME="/root/.local/pipx"
|
||||
ENV PIPX_BIN_DIR="/root/.local/bin"
|
||||
# 禁止 pip 隱性下載（強制離線模式在安裝完成後啟用）
|
||||
ENV PIP_NO_CACHE_DIR=1
|
||||
# HuggingFace 環境變數（安裝時允許下載，安裝完成後設為離線）
|
||||
ENV HF_HOME="/root/.cache/huggingface"
|
||||
|
||||
# ==============================================================================
|
||||
# 🔥 模型預下載區塊（Docker Build 階段）
|
||||
# ==============================================================================
|
||||
#
|
||||
# ⚠️ 重要原則：
|
||||
# - 所有模型必須在 build 階段下載完成
|
||||
# - runtime 完全不依賴外部網路
|
||||
# - 禁止任何隱式下載行為
|
||||
#
|
||||
# 📦 預下載的模型清單：
|
||||
# 1. PDFMathTranslate / pdf2zh
|
||||
# - DocLayout-YOLO ONNX 模型（佈局分析）
|
||||
# - BabelDOC 相關資源（透過 --warmup）
|
||||
# 2. MinerU / magic-pdf
|
||||
# - DocLayout-YOLO（佈局分析）
|
||||
# - YOLOv8 MFD（公式偵測）
|
||||
# - UniMERNet（公式辨識）
|
||||
# - PaddleOCR（文字辨識）
|
||||
# - LayoutReader（閱讀順序）
|
||||
# - SLANet / UNet（表格辨識）
|
||||
#
|
||||
# ==============================================================================
|
||||
# 🔧 BuildKit 優化說明：
|
||||
# ==============================================================================
|
||||
# 解決 "no space left on device" 的核心策略：
|
||||
#
|
||||
# 1. 【單一 RUN 原則】
|
||||
# 所有模型下載 + cache 清理必須在同一個 RUN 中完成
|
||||
# 這樣 BuildKit 在計算 layer diff 時，只會看到「最終狀態」
|
||||
# 而不是「下載的 blob cache + 複製的模型」兩份資料
|
||||
#
|
||||
# 2. 【HuggingFace cache 必須刪除】
|
||||
# snapshot_download 會在 ~/.cache/huggingface/hub 下建立：
|
||||
# - blobs/：實際的模型檔案（用 SHA256 命名）
|
||||
# - snapshots/：指向 blobs 的 symlink 或複製
|
||||
# 當 local_dir_use_symlinks=False 時，檔案會被「複製」到目標目錄
|
||||
# 如果不刪除 cache，同一份模型會以兩份大小進入 layer diff
|
||||
#
|
||||
# 3. 【避免 overlayfs 重複壓縮】
|
||||
# exporting layers 時，BuildKit 會：
|
||||
# - 計算每層的 diff（新增/修改的檔案）
|
||||
# - 壓縮 diff 並寫入 /var/lib/buildkit/runc-overlayfs/
|
||||
# 如果 cache 沒刪，diff 會包含 cache + 目標目錄，壓縮時空間翻倍
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 階段 14-UNIFIED：所有模型下載 + 快取清理（單一 RUN 避免 layer 爆炸）
|
||||
# ------------------------------------------------------------------------------
|
||||
# 🔑 關鍵：這個 RUN 必須包含所有下載操作，並在結尾清理所有 cache
|
||||
# 這樣 overlayfs 的 diff 只包含「最終需要的模型檔案」
|
||||
# 而不是「cache 結構 + 模型副本」
|
||||
# ------------------------------------------------------------------------------
|
||||
RUN set -eux && \
|
||||
echo "===========================================================" && \
|
||||
echo "🚀 開始統一模型下載（單一 RUN 優化 BuildKit layer）" && \
|
||||
echo "🚀 階段 12-UNIFIED：Python 工具 + 模型統一安裝" && \
|
||||
echo "===========================================================" && \
|
||||
echo "⬇️ 此 RUN 包含所有 Docker build 階段下載" && \
|
||||
echo " Runtime 不會再下載任何資源" && \
|
||||
echo "===========================================================" && \
|
||||
\
|
||||
# ========================================
|
||||
# [1/5] PDFMathTranslate DocLayout-YOLO ONNX 模型
|
||||
# [1/8] 安裝 huggingface_hub（用於顯式模型下載）
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [1/5] 下載 DocLayout-YOLO ONNX 模型..." && \
|
||||
echo "📦 [1/8] 安裝 huggingface_hub..." && \
|
||||
pip3 install --no-cache-dir --break-system-packages huggingface_hub && \
|
||||
\
|
||||
# ========================================
|
||||
# [2/8] 安裝 markitdown（文件轉換工具）
|
||||
# ⬇️ Docker build 階段安裝，無隱性下載
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📦 [2/8] 安裝 markitdown[all]..." && \
|
||||
pipx install "markitdown[all]" && \
|
||||
\
|
||||
# ========================================
|
||||
# [3/8] 安裝 pdf2zh（PDFMathTranslate 引擎）
|
||||
# ⬇️ Docker build 階段安裝
|
||||
# ⚠️ 模型將在後續步驟顯式下載，此處僅安裝程式
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📦 [3/8] 安裝 pdf2zh..." && \
|
||||
pipx install "pdf2zh" && \
|
||||
\
|
||||
# ========================================
|
||||
# [4/8] 安裝 babeldoc（BabelDOC 引擎）
|
||||
# ⬇️ Docker build 階段安裝
|
||||
# ⚠️ 資源將在後續步驟顯式下載，禁止使用 --warmup
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📦 [4/8] 安裝 babeldoc..." && \
|
||||
(pipx install "babeldoc" || echo "⚠️ babeldoc 安裝失敗，跳過...") && \
|
||||
\
|
||||
# ========================================
|
||||
# [5/8] 安裝 mineru（MinerU 引擎）
|
||||
# ⬇️ Docker build 階段安裝
|
||||
# ⚠️ 模型將在後續步驟顯式下載，此處僅安裝程式
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📦 [5/8] 安裝 mineru[all]..." && \
|
||||
(pipx install "mineru[all]" || echo "⚠️ mineru 安裝失敗（可能是 arm64 相容性問題），跳過...") && \
|
||||
\
|
||||
# ========================================
|
||||
# [6/8] 顯式下載 PDFMathTranslate 模型
|
||||
# ⬇️ Docker build 階段下載 DocLayout-YOLO ONNX 模型
|
||||
# Runtime 不會再下載任何資源
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [6/8] 下載 PDFMathTranslate DocLayout-YOLO ONNX 模型..." && \
|
||||
mkdir -p /models/pdfmathtranslate && \
|
||||
python3 -c " \
|
||||
from huggingface_hub import snapshot_download; \
|
||||
snapshot_download( \
|
||||
repo_id='wybxc/DocLayout-YOLO-DocStructBench-onnx', \
|
||||
local_dir='/models/pdfmathtranslate', \
|
||||
allow_patterns=['*.onnx'], \
|
||||
local_dir_use_symlinks=False \
|
||||
)" && \
|
||||
echo "✅ DocLayout-YOLO ONNX 下載完成" && \
|
||||
python3 -c "from huggingface_hub import snapshot_download; import os; os.environ['HF_HOME']='/root/.cache/huggingface'; snapshot_download(repo_id='wybxc/DocLayout-YOLO-DocStructBench-onnx', local_dir='/models/pdfmathtranslate', allow_patterns=['*.onnx'], local_dir_use_symlinks=False); print('DocLayout-YOLO ONNX downloaded')" && \
|
||||
ls -lh /models/pdfmathtranslate/*.onnx 2>/dev/null || ls -lh /models/pdfmathtranslate/ && \
|
||||
\
|
||||
# 🔥 立即清理 HuggingFace cache（關鍵！避免 blob 重複）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
\
|
||||
# ========================================
|
||||
# [2/5] BabelDOC 資源預下載
|
||||
# ========================================
|
||||
# 策略：使用 --generate-offline-assets 生成離線包
|
||||
# 這比 --warmup 更穩定，因為可以完整驗證所有資源
|
||||
echo "" && \
|
||||
echo "📥 [2/5] 執行 BabelDOC 資源預下載..." && \
|
||||
if command -v babeldoc >/dev/null 2>&1; then \
|
||||
BABELDOC_MAX_RETRIES=3; \
|
||||
BABELDOC_RETRY_COUNT=0; \
|
||||
BABELDOC_SUCCESS=false; \
|
||||
mkdir -p /tmp/babeldoc-offline && \
|
||||
while [ $BABELDOC_RETRY_COUNT -lt $BABELDOC_MAX_RETRIES ]; do \
|
||||
BABELDOC_RETRY_COUNT=$((BABELDOC_RETRY_COUNT + 1)); \
|
||||
echo "🔄 BabelDOC 資源下載嘗試 $BABELDOC_RETRY_COUNT/$BABELDOC_MAX_RETRIES..."; \
|
||||
if timeout 600 babeldoc --generate-offline-assets /tmp/babeldoc-offline 2>&1; then \
|
||||
echo "✅ BabelDOC 離線資源包生成成功"; \
|
||||
OFFLINE_PKG=$(ls /tmp/babeldoc-offline/offline_assets_*.zip 2>/dev/null | head -1); \
|
||||
if [ -n "$OFFLINE_PKG" ] && [ -f "$OFFLINE_PKG" ]; then \
|
||||
echo "📦 找到離線包: $OFFLINE_PKG"; \
|
||||
if babeldoc --restore-offline-assets "$OFFLINE_PKG" 2>&1; then \
|
||||
echo "✅ BabelDOC 資源已成功恢復到快取"; \
|
||||
BABELDOC_SUCCESS=true; \
|
||||
break; \
|
||||
else \
|
||||
echo "⚠️ 資源恢復失敗，重試..."; \
|
||||
fi; \
|
||||
else \
|
||||
echo "⚠️ 未找到離線包，嘗試 warmup 模式..."; \
|
||||
if timeout 600 babeldoc --warmup 2>&1; then \
|
||||
BABELDOC_SUCCESS=true; \
|
||||
break; \
|
||||
fi; \
|
||||
fi; \
|
||||
else \
|
||||
echo "⚠️ BabelDOC 資源下載失敗或超時（10分鐘），等待 30 秒後重試..."; \
|
||||
sleep 30; \
|
||||
fi; \
|
||||
done; \
|
||||
rm -rf /tmp/babeldoc-offline; \
|
||||
if [ "$BABELDOC_SUCCESS" = "true" ]; then \
|
||||
echo "✅ BabelDOC 資源預下載完成"; \
|
||||
else \
|
||||
echo "⚠️ BabelDOC 資源下載在 $BABELDOC_MAX_RETRIES 次嘗試後仍失敗"; \
|
||||
echo " BabelDOC 功能將在 runtime 時按需下載資源"; \
|
||||
fi; \
|
||||
else \
|
||||
echo "⚠️ babeldoc 命令不存在，跳過資源預下載"; \
|
||||
fi && \
|
||||
echo "✅ BabelDOC 步驟完成" && \
|
||||
\
|
||||
# ========================================
|
||||
# [3/5] PDFMathTranslate 多語言字型
|
||||
# [6.1/8] 下載 PDFMathTranslate 多語言字型
|
||||
# ⬇️ Docker build 階段下載字型檔案
|
||||
# Runtime 不會再下載任何資源
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [3/5] 下載 PDFMathTranslate 多語言字型..." && \
|
||||
echo "📥 [6.1/8] 下載 PDFMathTranslate 多語言字型..." && \
|
||||
mkdir -p /app && \
|
||||
curl -fSL -o /app/GoNotoKurrent-Regular.ttf \
|
||||
"https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf" && \
|
||||
|
|
@ -406,69 +342,100 @@ RUN set -eux && \
|
|||
curl -fSL -o /app/SourceHanSerifKR-Regular.ttf \
|
||||
"https://github.com/timelic/source-han-serif/releases/download/main/SourceHanSerifKR-Regular.ttf" && \
|
||||
echo "✅ 字型下載完成" && \
|
||||
ls -lh /app/*.ttf && \
|
||||
\
|
||||
# ========================================
|
||||
# [4/5] MinerU Pipeline 模型
|
||||
# [7/8] 顯式下載 BabelDOC 資源
|
||||
# ⬇️ Docker build 階段顯式下載 BabelDOC 所需資源
|
||||
# ❌ 禁止使用 --warmup（不可控的隱性下載）
|
||||
# ✅ 使用 HuggingFace 顯式下載模型
|
||||
# Runtime 不會再下載任何資源
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [4/5] 下載 MinerU Pipeline 模型..." && \
|
||||
echo "📥 [7/8] 顯式下載 BabelDOC 資源..." && \
|
||||
mkdir -p /root/.cache/babeldoc/models && \
|
||||
mkdir -p /root/.cache/babeldoc/fonts && \
|
||||
\
|
||||
# 下載 BabelDOC 使用的 DocLayout-YOLO 模型（與 pdf2zh 共用）
|
||||
echo " 下載 BabelDOC DocLayout-YOLO 模型..." && \
|
||||
(python3 -c "from huggingface_hub import snapshot_download; import os; os.environ['HF_HOME']='/root/.cache/huggingface'; snapshot_download(repo_id='wybxc/DocLayout-YOLO-DocStructBench-onnx', local_dir='/root/.cache/babeldoc/models/doclayout-yolo', allow_patterns=['*.onnx'], local_dir_use_symlinks=False); print('BabelDOC DocLayout-YOLO downloaded')" || echo "BabelDOC DocLayout-YOLO skipped") && \
|
||||
(python3 -c "from huggingface_hub import snapshot_download; import os; os.environ['HF_HOME']='/root/.cache/huggingface'; snapshot_download(repo_id='funstory-ai/babeldoc-assets', local_dir='/root/.cache/babeldoc/assets', local_dir_use_symlinks=False); print('BabelDOC assets downloaded')" || echo "BabelDOC assets not available") && \
|
||||
\
|
||||
# 複製字型到 BabelDOC 目錄（避免 runtime 下載）
|
||||
echo " 複製字型到 BabelDOC 目錄..." && \
|
||||
cp /app/GoNotoKurrent-Regular.ttf /root/.cache/babeldoc/fonts/ 2>/dev/null || true && \
|
||||
cp /app/SourceHanSerifCN-Regular.ttf /root/.cache/babeldoc/fonts/ 2>/dev/null || true && \
|
||||
cp /app/SourceHanSerifTW-Regular.ttf /root/.cache/babeldoc/fonts/ 2>/dev/null || true && \
|
||||
cp /app/SourceHanSerifJP-Regular.ttf /root/.cache/babeldoc/fonts/ 2>/dev/null || true && \
|
||||
cp /app/SourceHanSerifKR-Regular.ttf /root/.cache/babeldoc/fonts/ 2>/dev/null || true && \
|
||||
echo "✅ BabelDOC 資源準備完成" && \
|
||||
\
|
||||
# ========================================
|
||||
# [8/8] 顯式下載 MinerU Pipeline 模型
|
||||
# ⬇️ Docker build 階段顯式下載 MinerU 所需模型
|
||||
# 使用 mineru-models-download CLI（如果可用）
|
||||
# 或使用 HuggingFace 顯式下載
|
||||
# Runtime 不會再下載任何資源
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [8/8] 下載 MinerU Pipeline 模型..." && \
|
||||
ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
echo "⚠️ ARM64 架構：MinerU 可能不完全支援，嘗試下載模型..."; \
|
||||
fi && \
|
||||
\
|
||||
# 方法 1：使用官方 CLI（如果可用）
|
||||
if command -v mineru-models-download >/dev/null 2>&1; then \
|
||||
echo "使用 mineru-models-download CLI..."; \
|
||||
mineru-models-download -s huggingface -m pipeline 2>&1 || true; \
|
||||
echo "mineru.json 內容："; \
|
||||
echo "使用 mineru-models-download CLI..." && \
|
||||
mineru-models-download -s huggingface -m pipeline 2>&1 || true && \
|
||||
echo "mineru.json 內容：" && \
|
||||
cat /root/mineru.json 2>/dev/null || echo "(未生成)"; \
|
||||
else \
|
||||
echo "mineru-models-download 不可用，跳過 MinerU 模型下載"; \
|
||||
echo "mineru-models-download 不可用，使用顯式 HuggingFace 下載..." && \
|
||||
mkdir -p /root/.cache/mineru/models && \
|
||||
(python3 -c "from huggingface_hub import snapshot_download; import os; os.environ['HF_HOME']='/root/.cache/huggingface'; snapshot_download(repo_id='opendatalab/PDF-Extract-Kit-1.0', local_dir='/root/.cache/mineru/models/PDF-Extract-Kit-1.0', local_dir_use_symlinks=False); print('PDF-Extract-Kit-1.0 downloaded')" || echo "MinerU model download failed") && \
|
||||
python3 -c "import json; config={'models-dir':{'pipeline':'/root/.cache/mineru/models/PDF-Extract-Kit-1.0','vlm':''},'model-source':'local','latex-delimiter-config':{'display':{'left':'@@','right':'@@'},'inline':{'left':'@','right':'@'}}}; f=open('/root/mineru.json','w'); json.dump(config,f,indent=2); f.close(); print('mineru.json generated')"; \
|
||||
fi && \
|
||||
echo "✅ MinerU 模型下載步驟完成" && \
|
||||
\
|
||||
# 🔥 再次清理 HuggingFace cache（MinerU 也會產生）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
\
|
||||
# ========================================
|
||||
# [5/5] 驗證 + mineru.json 補充
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [5/5] 驗證 MinerU 設定檔..." && \
|
||||
mkdir -p /root && \
|
||||
if [ -f /root/mineru.json ]; then \
|
||||
echo "✅ mineru.json 已由 mineru-models-download 生成"; \
|
||||
cat /root/mineru.json; \
|
||||
else \
|
||||
echo "⚠️ mineru.json 不存在，建立預設設定..."; \
|
||||
echo '{"models-dir":{"pipeline":"","vlm":""},"model-source":"huggingface","latex-delimiter-config":{"display":{"left":"$$","right":"$$"},"inline":{"left":"$","right":"$"}}}' > /root/mineru.json; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
\
|
||||
# ========================================
|
||||
# 🔥 最終 Cache 清理（關鍵！避免 overlayfs diff 爆炸）
|
||||
# ========================================
|
||||
# ⚠️ 此清理必須在同一個 RUN 內執行
|
||||
# 否則 cache 會進入 layer diff，導致 image 膨脹
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "===========================================================" && \
|
||||
echo "🧹 清理所有下載快取（降低 layer diff 大小）" && \
|
||||
echo "===========================================================" && \
|
||||
\
|
||||
# HuggingFace Hub cache（最大宗！包含所有 blob）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
\
|
||||
# pip / Python build cache
|
||||
rm -rf /root/.cache/pip && \
|
||||
rm -rf /root/.cache/uv && \
|
||||
\
|
||||
# pipx cache
|
||||
rm -rf /root/.local/pipx/.cache && \
|
||||
\
|
||||
# 通用 cache 目錄
|
||||
rm -rf /tmp/* && \
|
||||
rm -rf /var/tmp/* && \
|
||||
\
|
||||
# Python bytecode cache（可選，節省少量空間）
|
||||
find /root/.local -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
|
||||
find /usr -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
|
||||
\
|
||||
# ========================================
|
||||
# 📋 模型檔案驗證
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "===========================================================" && \
|
||||
echo "📋 模型檔案驗證" && \
|
||||
echo "===========================================================" && \
|
||||
echo "" && \
|
||||
\
|
||||
echo "🔹 PDFMathTranslate 模型：" && \
|
||||
ONNX_COUNT=$(find /models/pdfmathtranslate -name "*.onnx" 2>/dev/null | wc -l) && \
|
||||
if [ "$ONNX_COUNT" -gt 0 ]; then \
|
||||
|
|
@ -478,31 +445,37 @@ RUN set -eux && \
|
|||
echo " ❌ /models/pdfmathtranslate 中沒有 ONNX 模型"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
\
|
||||
echo "🔹 PDFMathTranslate 字型：" && \
|
||||
ls -lh /app/*.ttf 2>/dev/null || echo " ⚠️ 無字型檔案" && \
|
||||
echo "" && \
|
||||
echo "🔹 BabelDOC 快取：" && \
|
||||
\
|
||||
echo "🔹 BabelDOC 資源：" && \
|
||||
if [ -d "/root/.cache/babeldoc" ]; then \
|
||||
echo " ✅ BabelDOC 快取目錄存在"; \
|
||||
echo " ✅ BabelDOC 資源目錄存在"; \
|
||||
du -sh /root/.cache/babeldoc 2>/dev/null || true; \
|
||||
ls -la /root/.cache/babeldoc/ 2>/dev/null || true; \
|
||||
else \
|
||||
echo " ⚠️ BabelDOC 快取目錄不存在（可能需要 runtime 下載）"; \
|
||||
echo " ⚠️ BabelDOC 資源目錄不存在"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
\
|
||||
echo "🔹 MinerU 模型目錄：" && \
|
||||
if [ -f /root/mineru.json ]; then \
|
||||
echo " ✅ mineru.json 存在"; \
|
||||
cat /root/mineru.json; \
|
||||
MINERU_PIPELINE_DIR=$(python3 -c "import json; f=open('/root/mineru.json'); d=json.load(f); print(d.get('models-dir',{}).get('pipeline',''))" 2>/dev/null || echo ""); \
|
||||
if [ -n "$MINERU_PIPELINE_DIR" ] && [ -d "$MINERU_PIPELINE_DIR" ]; then \
|
||||
echo " ✅ MinerU Pipeline 模型目錄存在: $MINERU_PIPELINE_DIR"; \
|
||||
du -sh "$MINERU_PIPELINE_DIR" 2>/dev/null || true; \
|
||||
else \
|
||||
echo " ⚠️ MinerU Pipeline 模型目錄不存在或未設定"; \
|
||||
echo " 設定路徑: ${MINERU_PIPELINE_DIR:-'(未設定)'}"; \
|
||||
fi; \
|
||||
else \
|
||||
echo " ⚠️ mineru.json 不存在（MinerU 未正確安裝）"; \
|
||||
echo " ⚠️ mineru.json 不存在"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
\
|
||||
echo "🔹 確認 HuggingFace cache 已清除：" && \
|
||||
if [ -d "/root/.cache/huggingface" ]; then \
|
||||
echo " ❌ 警告：HuggingFace cache 仍存在！"; \
|
||||
|
|
@ -511,8 +484,11 @@ RUN set -eux && \
|
|||
echo " ✅ HuggingFace cache 已清除"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
\
|
||||
echo "===========================================================" && \
|
||||
echo "✅ 模型下載完成，所有快取已清理" && \
|
||||
echo "✅ 階段 12-UNIFIED 完成：所有 Python 工具 + 模型已安裝" && \
|
||||
echo " 所有 cache 已清理，layer diff 最小化" && \
|
||||
echo " Runtime 不會再下載任何資源" && \
|
||||
echo "==========================================================="
|
||||
|
||||
# PDFMathTranslate 環境變數
|
||||
|
|
@ -522,16 +498,24 @@ ENV NOTO_FONT_PATH="/app/GoNotoKurrent-Regular.ttf"
|
|||
# BabelDOC 環境變數
|
||||
ENV BABELDOC_CACHE_PATH="/root/.cache/babeldoc"
|
||||
ENV BABELDOC_SERVICE="google"
|
||||
# 禁止 BabelDOC 自動下載（強制使用預下載資源）
|
||||
ENV BABELDOC_OFFLINE="1"
|
||||
|
||||
# MinerU 環境變數
|
||||
# 注意：如果 build 時模型下載成功，mineru.json 會設定為 local
|
||||
# 如果下載失敗，允許 runtime 從 huggingface 下載
|
||||
# ENV MINERU_MODEL_SOURCE="local" # 由 mineru.json 控制
|
||||
# ENV HF_HUB_OFFLINE="1" # 不強制離線，允許 fallback
|
||||
# 強制使用本地模型，禁止 runtime 下載
|
||||
ENV MINERU_MODEL_SOURCE="local"
|
||||
|
||||
# HuggingFace 離線模式（禁止 runtime 下載）
|
||||
# ⚠️ 此變數在所有模型下載完成後設定
|
||||
ENV HF_HUB_OFFLINE="1"
|
||||
ENV TRANSFORMERS_OFFLINE="1"
|
||||
|
||||
# ==============================================================================
|
||||
# 最終清理（模型下載完成後）
|
||||
# ==============================================================================
|
||||
# ⚠️ 此清理步驟獨立於模型下載 RUN，僅清理文件檔案
|
||||
# 模型相關 cache 已在上一個 RUN 中清除
|
||||
# ==============================================================================
|
||||
RUN rm -rf /usr/share/doc/texlive* \
|
||||
&& rm -rf /usr/share/texlive/texmf-dist/doc \
|
||||
&& rm -rf /usr/share/doc/* \
|
||||
|
|
@ -597,7 +581,43 @@ ENV QTWEBENGINE_CHROMIUM_FLAGS="--no-sandbox"
|
|||
ENV PANDOC_PDF_ENGINE=pdflatex
|
||||
# Node 環境
|
||||
ENV NODE_ENV=production
|
||||
# PDFMathTranslate 預設翻譯服務（可透過環境變數覆寫）
|
||||
|
||||
# ==============================================================================
|
||||
# 🌐 PDFMathTranslate 翻譯服務設定
|
||||
# ==============================================================================
|
||||
# ⚠️ 重要：PDFMathTranslate 的翻譯功能需要網路連接！
|
||||
# - DocLayout-YOLO ONNX 模型（已離線預下載）：用於佈局分析
|
||||
# - 翻譯服務（需要網路）：將文字翻譯成目標語言
|
||||
#
|
||||
# 支援的翻譯服務：
|
||||
# - google: Google Translate（免費，需網路）
|
||||
# - bing: Microsoft Bing Translator（免費，需網路）
|
||||
# - deepl: DeepL（需 API Key，需網路）
|
||||
# - ollama: 本地 Ollama LLM（可離線，需額外設定）
|
||||
#
|
||||
# 若要完全離線翻譯，請使用 ollama 並設定 OLLAMA_HOST
|
||||
# ==============================================================================
|
||||
ENV PDFMATHTRANSLATE_SERVICE="google"
|
||||
|
||||
# ==============================================================================
|
||||
# 🔒 Runtime 模型離線模式設定
|
||||
# ==============================================================================
|
||||
# ⚠️ 這些設定禁止 runtime 下載「模型」，但不影響翻譯 API 調用
|
||||
# PDFMathTranslate 使用的 Google/Bing 翻譯是線上 API，不是模型下載
|
||||
# ==============================================================================
|
||||
|
||||
# HuggingFace 模型離線（禁止下載新模型）
|
||||
ENV HF_HUB_OFFLINE="1"
|
||||
ENV TRANSFORMERS_OFFLINE="1"
|
||||
ENV HF_DATASETS_OFFLINE="1"
|
||||
|
||||
# 禁止 pip 安裝新套件
|
||||
ENV PIP_NO_INDEX="1"
|
||||
|
||||
# MinerU 強制使用本地模型
|
||||
ENV MINERU_MODEL_SOURCE="local"
|
||||
|
||||
# BabelDOC 模型離線模式
|
||||
ENV BABELDOC_OFFLINE="1"
|
||||
|
||||
ENTRYPOINT [ "bun", "run", "dist/src/index.js" ]
|
||||
|
|
|
|||
653
Dockerfile.v0.1.11
Normal file
653
Dockerfile.v0.1.11
Normal file
|
|
@ -0,0 +1,653 @@
|
|||
# ==============================================================================
|
||||
# ConvertX-CN 官方 Docker Image
|
||||
# 版本：v0.1.11
|
||||
# ==============================================================================
|
||||
#
|
||||
# 📦 Image 說明：
|
||||
# - 這是 ConvertX-CN 官方 Docker Hub Image 的生產 Dockerfile
|
||||
# - 已內建完整功能，無需額外擴充
|
||||
# - ⚠️ 所有模型已在 build 階段預下載，runtime 不依賴網路
|
||||
#
|
||||
# 🌍 內建語言支援：
|
||||
# - OCR: 英文、繁體中文、簡體中文、日文、韓文、德文、法文
|
||||
# - Locale: en_US, zh_TW, zh_CN, ja_JP, ko_KR, de_DE, fr_FR
|
||||
# - 字型: Noto CJK, Liberation, 標楷體
|
||||
# - LaTeX: CJK、德文、法文、阿拉伯語、希伯來語
|
||||
#
|
||||
# 🤖 預下載模型清單：
|
||||
# - PDFMathTranslate: DocLayout-YOLO ONNX（佈局分析）
|
||||
# - BabelDOC: 資源透過顯式 Python 腳本下載，禁止 warmup
|
||||
# - MinerU: PDF-Extract-Kit-1.0（Pipeline 模型）
|
||||
# 包含：DocLayout-YOLO, YOLOv8 MFD, UniMERNet, PaddleOCR, LayoutReader, SLANet
|
||||
#
|
||||
# 📊 Image 大小：約 8-12 GB（含模型）
|
||||
#
|
||||
# ⚠️ Base Image：使用 debian:bookworm（穩定版）
|
||||
# - 確保 Multi-Arch (amd64/arm64) 構建穩定性
|
||||
# - 避免 trixie (testing) 套件同步不穩定問題
|
||||
#
|
||||
# 🔒 OFFLINE-FIRST 設計原則：
|
||||
# 1. 所有下載行為只發生在 Docker build 階段
|
||||
# 2. Runtime 完全不依賴外部網路
|
||||
# 3. 禁止任何隱式下載（warmup、first-import、lazy-load）
|
||||
# 4. 所有 cache 在同一 RUN 內清除，避免 layer 膨脹
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
FROM debian:bookworm-slim AS base
|
||||
LABEL org.opencontainers.image.source="https://github.com/pi-docket/ConvertX-CN"
|
||||
LABEL org.opencontainers.image.description="ConvertX-CN - 精簡版檔案轉換服務"
|
||||
LABEL org.opencontainers.image.version="0.1.11"
|
||||
WORKDIR /app
|
||||
|
||||
# 配置 APT 重試機制（解決 Multi-Arch Build 時的網路不穩定問題）
|
||||
RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::http::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::https::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::ftp::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'DPkg::Lock::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries
|
||||
|
||||
# install bun
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
unzip \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# if architecture is arm64, use the arm64 version of bun
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
curl -fsSL -o bun-linux-aarch64.zip https://github.com/oven-sh/bun/releases/download/bun-v1.3.6/bun-linux-aarch64.zip; \
|
||||
else \
|
||||
curl -fsSL -o bun-linux-x64-baseline.zip https://github.com/oven-sh/bun/releases/download/bun-v1.3.6/bun-linux-x64-baseline.zip; \
|
||||
fi
|
||||
|
||||
RUN unzip -j bun-linux-*.zip -d /usr/local/bin && \
|
||||
rm bun-linux-*.zip && \
|
||||
chmod +x /usr/local/bin/bun
|
||||
|
||||
# install dependencies into temp directory
|
||||
# this will cache them and speed up future builds
|
||||
FROM base AS install
|
||||
RUN mkdir -p /temp/dev
|
||||
COPY package.json bun.lock /temp/dev/
|
||||
RUN cd /temp/dev && bun install --frozen-lockfile
|
||||
|
||||
# install with --production (exclude devDependencies)
|
||||
RUN mkdir -p /temp/prod
|
||||
COPY package.json bun.lock /temp/prod/
|
||||
RUN cd /temp/prod && bun install --frozen-lockfile --production
|
||||
|
||||
FROM base AS prerelease
|
||||
WORKDIR /app
|
||||
COPY --from=install /temp/dev/node_modules node_modules
|
||||
COPY . .
|
||||
|
||||
# ENV NODE_ENV=production
|
||||
RUN bun run build
|
||||
|
||||
# copy production dependencies and source code into final image
|
||||
FROM base AS release
|
||||
|
||||
# ==============================================================================
|
||||
# 依賴安裝（分段安裝，優化 Multi-Arch Build 穩定性）
|
||||
# ==============================================================================
|
||||
#
|
||||
# ✅ 核心轉換工具：完整保留
|
||||
# ✅ TexLive：完整 CJK + 德法 + 阿拉伯/希伯來語
|
||||
# ✅ OCR：7 種主要語言
|
||||
# ✅ 字型：Noto CJK + Liberation + 標楷體
|
||||
# ✅ OpenCV：電腦視覺轉換支援
|
||||
# ✅ 額外影片編解碼器
|
||||
# ✅ PDFMathTranslate：PDF 翻譯引擎
|
||||
#
|
||||
# 📝 分段安裝說明：
|
||||
# - 將套件拆分為多個 RUN 層，避免 QEMU 模擬時記憶體不足
|
||||
# - 每段安裝後清理 apt cache，減少中間層大小
|
||||
# - 最終 squash 時會合併為單一層
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
# 配置 APT 重試機制（解決 Multi-Arch Build 時的網路不穩定問題）
|
||||
RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::http::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::https::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'Acquire::ftp::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf.d/80-retries \
|
||||
&& echo 'DPkg::Lock::Timeout "120";' >> /etc/apt/apt.conf.d/80-retries
|
||||
|
||||
# 階段 1：基礎系統工具
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
locales \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 2：核心轉換工具（小型）
|
||||
# 注意：dasel 和 resvg 在 bookworm 中不存在，後續用二進位檔案安裝
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
assimp-utils \
|
||||
dcraw \
|
||||
dvisvgm \
|
||||
ghostscript \
|
||||
graphicsmagick \
|
||||
mupdf-tools \
|
||||
poppler-utils \
|
||||
potrace \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 2.1：安裝 dasel（從 GitHub 下載二進位檔案）
|
||||
# ⬇️ Docker build 階段下載，runtime 不會再下載
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
DASEL_ARCH="linux_arm64"; \
|
||||
else \
|
||||
DASEL_ARCH="linux_amd64"; \
|
||||
fi && \
|
||||
curl -sSLf "https://github.com/TomWright/dasel/releases/download/v2.8.1/dasel_${DASEL_ARCH}" -o /usr/local/bin/dasel && \
|
||||
chmod +x /usr/local/bin/dasel
|
||||
|
||||
# 階段 2.2：安裝 resvg（從 GitHub 下載二進位檔案）
|
||||
# 注意：resvg 官方只提供 x86_64 版本，ARM64 需從源碼編譯或跳過
|
||||
# ⬇️ Docker build 階段下載，runtime 不會再下載
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
echo "⚠️ resvg 沒有 ARM64 預編譯版本，跳過安裝（可改用 ImageMagick 或 Inkscape 替代）"; \
|
||||
else \
|
||||
curl -sSLf "https://github.com/linebender/resvg/releases/download/v0.44.0/resvg-linux-x86_64.tar.gz" -o /tmp/resvg.tar.gz && \
|
||||
tar -xzf /tmp/resvg.tar.gz -C /tmp/ && \
|
||||
mv /tmp/resvg /usr/local/bin/resvg && \
|
||||
chmod +x /usr/local/bin/resvg && \
|
||||
rm -rf /tmp/resvg.tar.gz; \
|
||||
fi
|
||||
|
||||
# 階段 3：影音處理工具
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libavcodec-extra \
|
||||
libva2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 4：圖像處理工具
|
||||
# 注意：bookworm 使用 imagemagick（版本 6），trixie 才有 imagemagick-7
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
imagemagick \
|
||||
inkscape \
|
||||
libheif-examples \
|
||||
libjxl-tools \
|
||||
libvips-tools \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 5：文件處理工具
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
calibre \
|
||||
libemail-outlook-message-perl \
|
||||
pandoc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 6：LibreOffice（最大的套件，單獨安裝）
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
libreoffice \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 7：TexLive 基礎
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
texlive-base \
|
||||
texlive-latex-base \
|
||||
texlive-latex-recommended \
|
||||
texlive-fonts-recommended \
|
||||
texlive-xetex \
|
||||
latexmk \
|
||||
lmodern \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 8：TexLive 語言包
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
texlive-lang-cjk \
|
||||
texlive-lang-german \
|
||||
texlive-lang-french \
|
||||
texlive-lang-arabic \
|
||||
texlive-lang-other \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 9：OCR 支援
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-eng \
|
||||
tesseract-ocr-chi-tra \
|
||||
tesseract-ocr-chi-sim \
|
||||
tesseract-ocr-jpn \
|
||||
tesseract-ocr-kor \
|
||||
tesseract-ocr-deu \
|
||||
tesseract-ocr-fra \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 10：字型
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
fonts-noto-cjk \
|
||||
fonts-noto-core \
|
||||
fonts-noto-color-emoji \
|
||||
fonts-liberation \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 階段 11：Python 依賴
|
||||
RUN apt-get update --fix-missing && apt-get install -y --no-install-recommends \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-numpy \
|
||||
python3-tinycss2 \
|
||||
python3-opencv \
|
||||
pipx \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ==============================================================================
|
||||
# 🔧 Python 工具安裝策略
|
||||
# ==============================================================================
|
||||
#
|
||||
# ⚠️ 重要原則：
|
||||
# 1. pipx install 只安裝 CLI 工具本身
|
||||
# 2. 模型/資源下載必須在後續單獨的 RUN 中顯式執行
|
||||
# 3. 禁止依賴 --warmup 或 first-import 隱式下載
|
||||
# 4. 所有 cache 必須在同一 RUN 內清除
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
# 階段 12：安裝 Python 基礎工具
|
||||
# ⬇️ 此步驟只安裝 huggingface_hub 用於後續模型下載
|
||||
# runtime 不會再使用此套件下載任何資源
|
||||
RUN pip3 install --no-cache-dir --break-system-packages huggingface_hub \
|
||||
&& rm -rf /root/.cache/pip /tmp/*
|
||||
|
||||
# 階段 12-A：安裝 markitdown（文件轉換工具）
|
||||
# 注意：markitdown[all] 可能依賴 transformers，但不會在 import 時下載模型
|
||||
# ⬇️ 此步驟為 Docker build 階段安裝，runtime 不會再下載
|
||||
RUN pipx install "markitdown[all]" \
|
||||
&& rm -rf /root/.cache/pip /root/.local/pipx/.cache /tmp/*
|
||||
|
||||
# 階段 12-B：安裝 pdf2zh（PDFMathTranslate 引擎）
|
||||
# 注意：pdf2zh 的模型將在後續階段顯式下載
|
||||
# ⬇️ 此步驟為 Docker build 階段安裝，runtime 不會再下載
|
||||
RUN pipx install "pdf2zh" \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /root/.local/pipx/.cache /tmp/*
|
||||
|
||||
# 階段 12-C：安裝 babeldoc（BabelDOC 引擎）
|
||||
# ⚠️ 注意：不使用 --warmup，資源將在後續階段顯式下載
|
||||
# ⬇️ 此步驟為 Docker build 階段安裝，runtime 不會再下載
|
||||
RUN (pipx install "babeldoc" || echo "⚠️ babeldoc 安裝失敗，跳過...") \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /root/.local/pipx/.cache /tmp/*
|
||||
|
||||
# 階段 12-D：安裝 mineru（MinerU PDF 解析引擎）
|
||||
# ⚠️ 注意：mineru[all] 依賴大量 HuggingFace 套件，但安裝時不應下載模型
|
||||
# 模型下載將在後續階段顯式執行
|
||||
# ⬇️ 此步驟為 Docker build 階段安裝，runtime 不會再下載
|
||||
RUN (pipx install "mineru[all]" || echo "⚠️ mineru 安裝失敗（可能是 arm64 相容性問題），跳過...") \
|
||||
&& rm -rf /root/.cache/huggingface /root/.cache/pip /root/.cache/torch /root/.local/pipx/.cache /tmp/*
|
||||
|
||||
# Add pipx bin directory to PATH（必須在模型下載前設定）
|
||||
ENV PATH="/root/.local/bin:${PATH}"
|
||||
|
||||
# ==============================================================================
|
||||
# 🔥 模型預下載區塊（Docker Build 階段）
|
||||
# ==============================================================================
|
||||
#
|
||||
# ⚠️ 核心原則（OFFLINE-FIRST）：
|
||||
# 1. 所有模型必須在 build 階段下載完成
|
||||
# 2. Runtime 完全不依賴外部網路
|
||||
# 3. 禁止任何隱式下載行為（warmup、lazy-load、first-import）
|
||||
# 4. 使用顯式 Python 腳本下載，而非 CLI warmup
|
||||
# 5. 所有 cache 必須在同一 RUN 內清除
|
||||
#
|
||||
# 📦 預下載的模型清單：
|
||||
# 1. PDFMathTranslate / pdf2zh
|
||||
# - DocLayout-YOLO ONNX 模型（佈局分析）
|
||||
# 2. MinerU / magic-pdf
|
||||
# - DocLayout-YOLO（佈局分析）
|
||||
# - YOLOv8 MFD（公式偵測）
|
||||
# - UniMERNet（公式辨識）
|
||||
# - PaddleOCR（文字辨識）
|
||||
# - LayoutReader（閱讀順序）
|
||||
# - SLANet / UNet（表格辨識）
|
||||
# 3. BabelDOC
|
||||
# - 翻譯服務不需要本地模型（使用 API）
|
||||
# - 字型和資源透過顯式下載
|
||||
#
|
||||
# ==============================================================================
|
||||
# 🔧 BuildKit 優化說明：
|
||||
# ==============================================================================
|
||||
# 解決 "no space left on device" 的核心策略：
|
||||
#
|
||||
# 1. 【單一 RUN 原則】
|
||||
# 所有模型下載 + cache 清理必須在同一個 RUN 中完成
|
||||
# 這樣 BuildKit 在計算 layer diff 時，只會看到「最終狀態」
|
||||
# 而不是「下載的 blob cache + 複製的模型」兩份資料
|
||||
#
|
||||
# 2. 【HuggingFace cache 必須刪除】
|
||||
# snapshot_download 會在 ~/.cache/huggingface/hub 下建立：
|
||||
# - blobs/：實際的模型檔案（用 SHA256 命名）
|
||||
# - snapshots/：指向 blobs 的 symlink 或複製
|
||||
# 當 local_dir_use_symlinks=False 時，檔案會被「複製」到目標目錄
|
||||
# 如果不刪除 cache，同一份模型會以兩份大小進入 layer diff
|
||||
#
|
||||
# 3. 【避免 overlayfs 重複壓縮】
|
||||
# exporting layers 時，BuildKit 會：
|
||||
# - 計算每層的 diff（新增/修改的檔案）
|
||||
# - 壓縮 diff 並寫入 /var/lib/buildkit/runc-overlayfs/
|
||||
# 如果 cache 沒刪，diff 會包含 cache + 目標目錄，壓縮時空間翻倍
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# 階段 14-UNIFIED：所有模型下載 + 快取清理（單一 RUN 避免 layer 爆炸）
|
||||
# ------------------------------------------------------------------------------
|
||||
# 🔑 關鍵：這個 RUN 必須包含所有下載操作，並在結尾清理所有 cache
|
||||
# 這樣 overlayfs 的 diff 只包含「最終需要的模型檔案」
|
||||
# 而不是「cache 結構 + 模型副本」
|
||||
#
|
||||
# ⬇️ 以下為 Docker build 階段下載所有模型
|
||||
# runtime 不會再下載任何資源
|
||||
# ------------------------------------------------------------------------------
|
||||
RUN set -eux && \
|
||||
echo "===========================================================" && \
|
||||
echo "🚀 開始統一模型下載（單一 RUN 優化 BuildKit layer）" && \
|
||||
echo "===========================================================" && \
|
||||
\
|
||||
# ========================================
|
||||
# [1/5] PDFMathTranslate DocLayout-YOLO ONNX 模型
|
||||
# ========================================
|
||||
# ⬇️ Docker build 階段下載 DocLayout-YOLO ONNX 模型
|
||||
# 這是 pdf2zh 用於 PDF 版面分析的核心模型
|
||||
# runtime 不會再下載此資源
|
||||
echo "" && \
|
||||
echo "📥 [1/5] 下載 DocLayout-YOLO ONNX 模型..." && \
|
||||
mkdir -p /models/pdfmathtranslate && \
|
||||
python3 -c " \
|
||||
import os; \
|
||||
os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '0'; \
|
||||
from huggingface_hub import snapshot_download; \
|
||||
snapshot_download( \
|
||||
repo_id='wybxc/DocLayout-YOLO-DocStructBench-onnx', \
|
||||
local_dir='/models/pdfmathtranslate', \
|
||||
allow_patterns=['*.onnx'], \
|
||||
local_dir_use_symlinks=False \
|
||||
)" && \
|
||||
echo "✅ DocLayout-YOLO ONNX 下載完成" && \
|
||||
ls -lh /models/pdfmathtranslate/*.onnx 2>/dev/null || ls -lh /models/pdfmathtranslate/ && \
|
||||
\
|
||||
# 🔥 立即清理 HuggingFace cache（關鍵！避免 blob 重複進入 layer）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
\
|
||||
# ========================================
|
||||
# [2/5] BabelDOC 資源（顯式下載，禁止 warmup）
|
||||
# ========================================
|
||||
# ⚠️ 重要：不使用 babeldoc --warmup，因為：
|
||||
# 1. --warmup 內部使用 async HTTP，在 QEMU 下不穩定
|
||||
# 2. --warmup 可能失敗但不報錯
|
||||
# 3. 我們需要完全可控的下載流程
|
||||
#
|
||||
# BabelDOC 翻譯服務使用外部 API（Google/DeepL/OpenAI）
|
||||
# 本地只需要字型和基礎資源，這些已透過 apt 安裝
|
||||
# ⬇️ Docker build 階段準備 BabelDOC 快取目錄
|
||||
# runtime 使用 API 翻譯，不需要本地模型
|
||||
echo "" && \
|
||||
echo "📥 [2/5] 準備 BabelDOC 資源..." && \
|
||||
mkdir -p /root/.cache/babeldoc && \
|
||||
if command -v babeldoc >/dev/null 2>&1; then \
|
||||
echo "✅ BabelDOC 已安裝，翻譯將使用外部 API（Google/DeepL/OpenAI）"; \
|
||||
echo " 本地不需要下載翻譯模型"; \
|
||||
else \
|
||||
echo "⚠️ BabelDOC 未安裝，跳過"; \
|
||||
fi && \
|
||||
echo "✅ BabelDOC 步驟完成" && \
|
||||
\
|
||||
# ========================================
|
||||
# [3/5] PDFMathTranslate 多語言字型
|
||||
# ========================================
|
||||
# ⬇️ Docker build 階段下載 PDFMathTranslate 所需字型
|
||||
# 這些字型用於 PDF 翻譯時保持正確的文字渲染
|
||||
# runtime 不會再下載此資源
|
||||
echo "" && \
|
||||
echo "📥 [3/5] 下載 PDFMathTranslate 多語言字型..." && \
|
||||
mkdir -p /app && \
|
||||
curl -fSL --retry 3 --retry-delay 5 -o /app/GoNotoKurrent-Regular.ttf \
|
||||
"https://github.com/satbyy/go-noto-universal/releases/download/v7.0/GoNotoKurrent-Regular.ttf" && \
|
||||
curl -fSL --retry 3 --retry-delay 5 -o /app/SourceHanSerifCN-Regular.ttf \
|
||||
"https://github.com/timelic/source-han-serif/releases/download/main/SourceHanSerifCN-Regular.ttf" && \
|
||||
curl -fSL --retry 3 --retry-delay 5 -o /app/SourceHanSerifTW-Regular.ttf \
|
||||
"https://github.com/timelic/source-han-serif/releases/download/main/SourceHanSerifTW-Regular.ttf" && \
|
||||
curl -fSL --retry 3 --retry-delay 5 -o /app/SourceHanSerifJP-Regular.ttf \
|
||||
"https://github.com/timelic/source-han-serif/releases/download/main/SourceHanSerifJP-Regular.ttf" && \
|
||||
curl -fSL --retry 3 --retry-delay 5 -o /app/SourceHanSerifKR-Regular.ttf \
|
||||
"https://github.com/timelic/source-han-serif/releases/download/main/SourceHanSerifKR-Regular.ttf" && \
|
||||
echo "✅ 字型下載完成" && \
|
||||
\
|
||||
# ========================================
|
||||
# [4/5] MinerU Pipeline 模型（顯式下載）
|
||||
# ========================================
|
||||
# ⬇️ Docker build 階段下載 MinerU Pipeline 模型
|
||||
# 包含：DocLayout-YOLO, YOLOv8 MFD, UniMERNet, PaddleOCR, LayoutReader, SLANet
|
||||
# runtime 不會再下載任何資源
|
||||
echo "" && \
|
||||
echo "📥 [4/5] 下載 MinerU Pipeline 模型..." && \
|
||||
ARCH=$(uname -m) && \
|
||||
MINERU_MODELS_DIR="/models/mineru" && \
|
||||
mkdir -p "$MINERU_MODELS_DIR" && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
echo "⚠️ ARM64 架構：MinerU 可能不完全支援，嘗試下載模型..."; \
|
||||
fi && \
|
||||
if command -v mineru >/dev/null 2>&1; then \
|
||||
echo "使用顯式 Python 腳本下載 MinerU 模型..."; \
|
||||
python3 -c " \
|
||||
import os, json; \
|
||||
os.environ['HF_HUB_DISABLE_PROGRESS_BARS'] = '0'; \
|
||||
from huggingface_hub import snapshot_download; \
|
||||
# PDF-Extract-Kit-1.0 Pipeline 模型
|
||||
models_dir = '/models/mineru'; \
|
||||
print('下載 PDF-Extract-Kit-1.0 模型...'); \
|
||||
try: \
|
||||
snapshot_download( \
|
||||
repo_id='opendatalab/PDF-Extract-Kit-1.0', \
|
||||
local_dir=os.path.join(models_dir, 'PDF-Extract-Kit-1.0'), \
|
||||
local_dir_use_symlinks=False \
|
||||
); \
|
||||
print('✅ PDF-Extract-Kit-1.0 下載完成'); \
|
||||
except Exception as e: \
|
||||
print(f'⚠️ PDF-Extract-Kit-1.0 下載失敗: {e}'); \
|
||||
# 建立 mineru.json 設定檔
|
||||
config = { \
|
||||
'models-dir': { \
|
||||
'pipeline': os.path.join(models_dir, 'PDF-Extract-Kit-1.0'), \
|
||||
'vlm': '' \
|
||||
}, \
|
||||
'model-source': 'local', \
|
||||
'latex-delimiter-config': { \
|
||||
'display': {'left': '\$\$', 'right': '\$\$'}, \
|
||||
'inline': {'left': '\$', 'right': '\$'} \
|
||||
} \
|
||||
}; \
|
||||
with open('/root/mineru.json', 'w') as f: \
|
||||
json.dump(config, f, indent=2); \
|
||||
print('✅ mineru.json 已建立'); \
|
||||
" || echo "⚠️ MinerU 模型下載失敗，將在 runtime 時嘗試下載"; \
|
||||
else \
|
||||
echo "⚠️ mineru 命令不可用，跳過模型下載"; \
|
||||
echo '{"models-dir":{"pipeline":"","vlm":""},"model-source":"huggingface","latex-delimiter-config":{"display":{"left":"$$","right":"$$"},"inline":{"left":"$","right":"$"}}}' > /root/mineru.json; \
|
||||
fi && \
|
||||
echo "✅ MinerU 模型下載步驟完成" && \
|
||||
\
|
||||
# 🔥 再次清理 HuggingFace cache（MinerU 也會產生）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
\
|
||||
# ========================================
|
||||
# [5/5] 最終驗證
|
||||
# ========================================
|
||||
echo "" && \
|
||||
echo "📥 [5/5] 驗證模型完整性..." && \
|
||||
echo "" && \
|
||||
\
|
||||
# ========================================
|
||||
# 🔥 最終 Cache 清理（關鍵！避免 overlayfs diff 爆炸）
|
||||
# ========================================
|
||||
echo "===========================================================" && \
|
||||
echo "🧹 清理所有下載快取（降低 layer diff 大小）" && \
|
||||
echo "===========================================================" && \
|
||||
# HuggingFace Hub cache（最大宗！包含所有 blob）
|
||||
rm -rf /root/.cache/huggingface && \
|
||||
# pip / Python build cache
|
||||
rm -rf /root/.cache/pip && \
|
||||
rm -rf /root/.cache/uv && \
|
||||
# pipx cache
|
||||
rm -rf /root/.local/pipx/.cache && \
|
||||
# torch cache（可能由 MinerU 產生）
|
||||
rm -rf /root/.cache/torch && \
|
||||
# 通用 cache 目錄
|
||||
rm -rf /tmp/* && \
|
||||
rm -rf /var/tmp/* && \
|
||||
# Python bytecode cache（可選，節省少量空間）
|
||||
find /root/.local -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
|
||||
find /usr -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true && \
|
||||
\
|
||||
echo "" && \
|
||||
echo "===========================================================" && \
|
||||
echo "📋 模型檔案驗證" && \
|
||||
echo "===========================================================" && \
|
||||
echo "" && \
|
||||
echo "🔹 PDFMathTranslate 模型：" && \
|
||||
ONNX_COUNT=$(find /models/pdfmathtranslate -name "*.onnx" 2>/dev/null | wc -l) && \
|
||||
if [ "$ONNX_COUNT" -gt 0 ]; then \
|
||||
echo " ✅ 找到 $ONNX_COUNT 個 ONNX 模型："; \
|
||||
ls -lh /models/pdfmathtranslate/*.onnx 2>/dev/null || find /models/pdfmathtranslate -name "*.onnx" -exec ls -lh {} \;; \
|
||||
else \
|
||||
echo " ❌ /models/pdfmathtranslate 中沒有 ONNX 模型"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
echo "🔹 PDFMathTranslate 字型：" && \
|
||||
ls -lh /app/*.ttf 2>/dev/null || echo " ⚠️ 無字型檔案" && \
|
||||
echo "" && \
|
||||
echo "🔹 BabelDOC 狀態：" && \
|
||||
if command -v babeldoc >/dev/null 2>&1; then \
|
||||
echo " ✅ BabelDOC 已安裝（使用外部 API 翻譯）"; \
|
||||
else \
|
||||
echo " ⚠️ BabelDOC 未安裝"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
echo "🔹 MinerU 模型目錄：" && \
|
||||
if [ -d "/models/mineru/PDF-Extract-Kit-1.0" ]; then \
|
||||
echo " ✅ MinerU Pipeline 模型已下載"; \
|
||||
du -sh /models/mineru/PDF-Extract-Kit-1.0 2>/dev/null || true; \
|
||||
else \
|
||||
echo " ⚠️ MinerU 模型目錄不存在（可能需要 runtime 下載）"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
echo "🔹 mineru.json 設定：" && \
|
||||
if [ -f /root/mineru.json ]; then \
|
||||
cat /root/mineru.json; \
|
||||
else \
|
||||
echo " ⚠️ mineru.json 不存在"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
echo "🔹 確認 HuggingFace cache 已清除：" && \
|
||||
if [ -d "/root/.cache/huggingface" ]; then \
|
||||
echo " ❌ 警告：HuggingFace cache 仍存在！"; \
|
||||
du -sh /root/.cache/huggingface 2>/dev/null || true; \
|
||||
else \
|
||||
echo " ✅ HuggingFace cache 已清除"; \
|
||||
fi && \
|
||||
echo "" && \
|
||||
echo "===========================================================" && \
|
||||
echo "✅ 模型下載完成，所有快取已清理" && \
|
||||
echo "==========================================================="
|
||||
|
||||
# ==============================================================================
|
||||
# 環境變數設定
|
||||
# ==============================================================================
|
||||
|
||||
# PDFMathTranslate 環境變數
|
||||
ENV PDFMATHTRANSLATE_MODELS_PATH="/models/pdfmathtranslate"
|
||||
ENV NOTO_FONT_PATH="/app/GoNotoKurrent-Regular.ttf"
|
||||
|
||||
# BabelDOC 環境變數
|
||||
ENV BABELDOC_CACHE_PATH="/root/.cache/babeldoc"
|
||||
ENV BABELDOC_SERVICE="google"
|
||||
|
||||
# MinerU 環境變數
|
||||
# 強制使用本地模型，禁止 runtime 下載
|
||||
ENV MINERU_MODELS_PATH="/models/mineru"
|
||||
|
||||
# HuggingFace 離線模式（禁止 runtime 下載）
|
||||
# ⚠️ 這是 OFFLINE-FIRST 設計的核心
|
||||
ENV HF_HUB_OFFLINE="1"
|
||||
ENV TRANSFORMERS_OFFLINE="1"
|
||||
|
||||
# ==============================================================================
|
||||
# 最終清理（模型下載完成後）
|
||||
# ==============================================================================
|
||||
RUN rm -rf /usr/share/doc/texlive* \
|
||||
&& rm -rf /usr/share/texlive/texmf-dist/doc \
|
||||
&& rm -rf /usr/share/doc/* \
|
||||
&& rm -rf /usr/share/man/* \
|
||||
&& rm -rf /usr/share/info/*
|
||||
|
||||
# ==============================================================================
|
||||
# 設定 locale（支援中文 PDF 避免亂碼）
|
||||
# ==============================================================================
|
||||
RUN sed -i 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# zh_TW.UTF-8 UTF-8/zh_TW.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# zh_CN.UTF-8 UTF-8/zh_CN.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# ja_JP.UTF-8 UTF-8/ja_JP.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# ko_KR.UTF-8 UTF-8/ko_KR.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# de_DE.UTF-8 UTF-8/de_DE.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
sed -i 's/# fr_FR.UTF-8 UTF-8/fr_FR.UTF-8 UTF-8/' /etc/locale.gen && \
|
||||
locale-gen
|
||||
|
||||
# 預設使用 zh_TW.UTF-8 確保中文 PDF 正確顯示
|
||||
ENV LANG=zh_TW.UTF-8
|
||||
ENV LC_ALL=zh_TW.UTF-8
|
||||
|
||||
# ==============================================================================
|
||||
# 安裝自訂字型（標楷體等台灣常用字型）
|
||||
# ==============================================================================
|
||||
RUN mkdir -p /usr/share/fonts/truetype/custom
|
||||
COPY fonts/ /usr/share/fonts/truetype/custom/
|
||||
RUN fc-cache -fv
|
||||
|
||||
# ==============================================================================
|
||||
# Install VTracer binary（向量追蹤工具）
|
||||
# ⬇️ Docker build 階段下載，runtime 不會再下載
|
||||
# ==============================================================================
|
||||
RUN ARCH=$(uname -m) && \
|
||||
if [ "$ARCH" = "aarch64" ]; then \
|
||||
VTRACER_ASSET="vtracer-aarch64-unknown-linux-musl.tar.gz"; \
|
||||
else \
|
||||
VTRACER_ASSET="vtracer-x86_64-unknown-linux-musl.tar.gz"; \
|
||||
fi && \
|
||||
curl -L --retry 3 --retry-delay 5 -o /tmp/vtracer.tar.gz "https://github.com/visioncortex/vtracer/releases/download/0.6.4/${VTRACER_ASSET}" && \
|
||||
tar -xzf /tmp/vtracer.tar.gz -C /tmp/ && \
|
||||
mv /tmp/vtracer /usr/local/bin/vtracer && \
|
||||
chmod +x /usr/local/bin/vtracer && \
|
||||
rm /tmp/vtracer.tar.gz
|
||||
|
||||
COPY --from=install /temp/prod/node_modules node_modules
|
||||
COPY --from=prerelease /app/public/ /app/public/
|
||||
COPY --from=prerelease /app/dist /app/dist
|
||||
|
||||
# 複製模型驗證腳本
|
||||
COPY scripts/verify-models.sh /app/scripts/verify-models.sh
|
||||
RUN chmod +x /app/scripts/verify-models.sh
|
||||
|
||||
RUN mkdir data
|
||||
|
||||
EXPOSE 3000/tcp
|
||||
|
||||
# ==============================================================================
|
||||
# 環境變數
|
||||
# ==============================================================================
|
||||
# Calibre 需要
|
||||
ENV QTWEBENGINE_CHROMIUM_FLAGS="--no-sandbox"
|
||||
# Pandoc PDF 引擎（使用 pdflatex 以獲得最佳相容性）
|
||||
ENV PANDOC_PDF_ENGINE=pdflatex
|
||||
# Node 環境
|
||||
ENV NODE_ENV=production
|
||||
# PDFMathTranslate 預設翻譯服務（可透過環境變數覆寫）
|
||||
ENV PDFMATHTRANSLATE_SERVICE="google"
|
||||
|
||||
ENTRYPOINT [ "bun", "run", "dist/src/index.js" ]
|
||||
220
scripts/verify-models.v0.1.11.sh
Normal file
220
scripts/verify-models.v0.1.11.sh
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# ConvertX-CN 模型驗證腳本
|
||||
# 版本：v0.1.11
|
||||
# ==============================================================================
|
||||
#
|
||||
# 用途：驗證 Docker Image 中的預下載模型是否完整
|
||||
# 執行：docker exec <container> /app/scripts/verify-models.sh
|
||||
#
|
||||
# ⚠️ 重要說明：
|
||||
# 此腳本驗證的是「模型目錄」而非「HuggingFace cache」
|
||||
# 因為 Dockerfile 中已將 cache 清除，模型被複製到獨立目錄
|
||||
#
|
||||
# ==============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "========================================"
|
||||
echo "🔍 ConvertX-CN 模型驗證 (v0.1.11)"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# 計數器
|
||||
PASS=0
|
||||
FAIL=0
|
||||
WARN=0
|
||||
|
||||
# 檢查函數
|
||||
check_file() {
|
||||
local path="$1"
|
||||
local name="$2"
|
||||
if [ -f "$path" ]; then
|
||||
local size=$(ls -lh "$path" | awk '{print $5}')
|
||||
echo "✅ $name: $path ($size)"
|
||||
((PASS++))
|
||||
else
|
||||
echo "❌ $name: $path 不存在"
|
||||
((FAIL++))
|
||||
fi
|
||||
}
|
||||
|
||||
check_dir() {
|
||||
local path="$1"
|
||||
local name="$2"
|
||||
local required="$3" # "required" 或 "optional"
|
||||
if [ -d "$path" ]; then
|
||||
local size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
|
||||
echo "✅ $name: $path ($size)"
|
||||
((PASS++))
|
||||
else
|
||||
if [ "$required" = "required" ]; then
|
||||
echo "❌ $name: $path 不存在"
|
||||
((FAIL++))
|
||||
else
|
||||
echo "⚠️ $name: $path 不存在（可選）"
|
||||
((WARN++))
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_command() {
|
||||
local cmd="$1"
|
||||
local name="$2"
|
||||
if command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "✅ $name: $(which $cmd)"
|
||||
((PASS++))
|
||||
else
|
||||
echo "⚠️ $name: 未安裝"
|
||||
((WARN++))
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 1. PDFMathTranslate 模型（必要）
|
||||
# ==============================================================================
|
||||
echo "📦 PDFMathTranslate 模型"
|
||||
echo "----------------------------------------"
|
||||
# 檢查 ONNX 模型（可能是 model.onnx 或其他名稱）
|
||||
ONNX_FILES=$(find /models/pdfmathtranslate -name "*.onnx" 2>/dev/null | wc -l)
|
||||
if [ "$ONNX_FILES" -gt 0 ]; then
|
||||
echo "✅ DocLayout-YOLO ONNX: 找到 $ONNX_FILES 個模型"
|
||||
find /models/pdfmathtranslate -name "*.onnx" -exec ls -lh {} \; 2>/dev/null
|
||||
((PASS++))
|
||||
else
|
||||
echo "❌ DocLayout-YOLO ONNX: 未找到任何 .onnx 檔案"
|
||||
((FAIL++))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 2. PDFMathTranslate 字型（必要）
|
||||
# ==============================================================================
|
||||
echo "🔤 PDFMathTranslate 字型"
|
||||
echo "----------------------------------------"
|
||||
check_file "/app/GoNotoKurrent-Regular.ttf" "Noto 通用字型"
|
||||
check_file "/app/SourceHanSerifCN-Regular.ttf" "思源宋體（簡體）"
|
||||
check_file "/app/SourceHanSerifTW-Regular.ttf" "思源宋體（繁體）"
|
||||
check_file "/app/SourceHanSerifJP-Regular.ttf" "思源宋體（日文）"
|
||||
check_file "/app/SourceHanSerifKR-Regular.ttf" "思源宋體（韓文）"
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 3. MinerU 模型（可選）
|
||||
# ==============================================================================
|
||||
echo "📦 MinerU 模型"
|
||||
echo "----------------------------------------"
|
||||
# 注意：模型現在位於 /models/mineru，而非 HuggingFace cache
|
||||
check_dir "/models/mineru/PDF-Extract-Kit-1.0" "PDF-Extract-Kit-1.0 Pipeline" "optional"
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 4. BabelDOC 狀態（可選）
|
||||
# ==============================================================================
|
||||
echo "📦 BabelDOC 狀態"
|
||||
echo "----------------------------------------"
|
||||
check_command "babeldoc" "BabelDOC CLI"
|
||||
if [ -d "/root/.cache/babeldoc" ]; then
|
||||
check_dir "/root/.cache/babeldoc" "BabelDOC 快取" "optional"
|
||||
else
|
||||
echo "ℹ️ BabelDOC 使用外部 API（Google/DeepL/OpenAI），不需要本地模型"
|
||||
((PASS++))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 5. CLI 工具檢查
|
||||
# ==============================================================================
|
||||
echo "🔧 CLI 工具"
|
||||
echo "----------------------------------------"
|
||||
check_command "pdf2zh" "PDFMathTranslate (pdf2zh)"
|
||||
check_command "mineru" "MinerU"
|
||||
check_command "markitdown" "MarkItDown"
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 6. 環境變數檢查
|
||||
# ==============================================================================
|
||||
echo "🔧 環境變數"
|
||||
echo "----------------------------------------"
|
||||
echo "PDFMATHTRANSLATE_MODELS_PATH: ${PDFMATHTRANSLATE_MODELS_PATH:-未設定}"
|
||||
echo "NOTO_FONT_PATH: ${NOTO_FONT_PATH:-未設定}"
|
||||
echo "MINERU_MODELS_PATH: ${MINERU_MODELS_PATH:-未設定}"
|
||||
echo "HF_HUB_OFFLINE: ${HF_HUB_OFFLINE:-未設定}"
|
||||
echo "TRANSFORMERS_OFFLINE: ${TRANSFORMERS_OFFLINE:-未設定}"
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 7. MinerU 設定檔
|
||||
# ==============================================================================
|
||||
echo "📝 MinerU 設定檔"
|
||||
echo "----------------------------------------"
|
||||
if [ -f "/root/mineru.json" ]; then
|
||||
echo "✅ /root/mineru.json 存在"
|
||||
cat /root/mineru.json
|
||||
((PASS++))
|
||||
else
|
||||
echo "⚠️ /root/mineru.json 不存在"
|
||||
((WARN++))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 8. Cache 清理驗證（OFFLINE-FIRST 設計要求）
|
||||
# ==============================================================================
|
||||
echo "🧹 Cache 清理驗證"
|
||||
echo "----------------------------------------"
|
||||
if [ -d "/root/.cache/huggingface" ]; then
|
||||
HF_SIZE=$(du -sh /root/.cache/huggingface 2>/dev/null | awk '{print $1}')
|
||||
echo "❌ HuggingFace cache 存在: $HF_SIZE"
|
||||
echo " 這違反 OFFLINE-FIRST 設計原則！"
|
||||
echo " cache 應該在 Docker build 時清除"
|
||||
((FAIL++))
|
||||
else
|
||||
echo "✅ HuggingFace cache 已清除（符合 OFFLINE-FIRST 設計）"
|
||||
((PASS++))
|
||||
fi
|
||||
|
||||
if [ -d "/root/.cache/pip" ]; then
|
||||
echo "⚠️ pip cache 存在（應該已清除）"
|
||||
((WARN++))
|
||||
else
|
||||
echo "✅ pip cache 已清除"
|
||||
((PASS++))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ==============================================================================
|
||||
# 總結
|
||||
# ==============================================================================
|
||||
echo "========================================"
|
||||
echo "📊 驗證結果"
|
||||
echo "========================================"
|
||||
echo "✅ 通過: $PASS"
|
||||
echo "⚠️ 警告: $WARN"
|
||||
echo "❌ 失敗: $FAIL"
|
||||
echo ""
|
||||
|
||||
if [ $FAIL -gt 0 ]; then
|
||||
echo "❌ 驗證失敗！部分必要模型缺失。"
|
||||
echo " 建議重新 build Docker Image。"
|
||||
exit 1
|
||||
elif [ $WARN -gt 0 ]; then
|
||||
echo "⚠️ 驗證完成，但有警告。"
|
||||
echo " 部分可選功能可能不可用。"
|
||||
echo ""
|
||||
echo "ℹ️ 提示："
|
||||
echo " - MinerU 模型可選，如需使用請確保 build 時下載成功"
|
||||
echo " - BabelDOC 使用外部 API，不需要本地模型"
|
||||
echo " - HF_HUB_OFFLINE=1 會阻止 runtime 下載"
|
||||
exit 0
|
||||
else
|
||||
echo "✅ 所有模型驗證通過！"
|
||||
echo " 系統可在離線環境正常運作。"
|
||||
echo ""
|
||||
echo "🔒 OFFLINE-FIRST 設計已生效："
|
||||
echo " - HF_HUB_OFFLINE=1"
|
||||
echo " - TRANSFORMERS_OFFLINE=1"
|
||||
echo " - 所有 cache 已清除"
|
||||
exit 0
|
||||
fi
|
||||
|
|
@ -468,9 +468,13 @@ export function convert(
|
|||
outputArgs.push("-background", "white", "-alpha", "remove");
|
||||
}
|
||||
|
||||
// 使用 convert（ImageMagick 6.x，Debian bookworm）
|
||||
// ImageMagick 7.x 使用 magick，但 Debian bookworm 只有 6.x
|
||||
const imCommand = process.env.IMAGEMAGICK_COMMAND || "convert";
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
"magick",
|
||||
imCommand,
|
||||
[...inputArgs, filePath, ...outputArgs, targetPath],
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
|
|
|
|||
|
|
@ -70,81 +70,115 @@ export async function convert(
|
|||
options?: unknown,
|
||||
execFile: ExecFileFn = execFileOriginal,
|
||||
): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Create a temporary output directory for MinerU
|
||||
const outputDir = dirname(targetPath);
|
||||
const inputFileName = basename(filePath, `.${fileType}`);
|
||||
const mineruOutputDir = join(outputDir, `${inputFileName}_mineru_${convertTo}`);
|
||||
// Create a temporary output directory for MinerU
|
||||
const outputDir = dirname(targetPath);
|
||||
const inputFileName = basename(filePath, `.${fileType}`);
|
||||
const mineruOutputDir = join(outputDir, `${inputFileName}_mineru_${convertTo}`);
|
||||
|
||||
// Ensure output directory exists
|
||||
if (!existsSync(mineruOutputDir)) {
|
||||
mkdirSync(mineruOutputDir, { recursive: true });
|
||||
}
|
||||
// Ensure output directory exists
|
||||
if (!existsSync(mineruOutputDir)) {
|
||||
mkdirSync(mineruOutputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Build MinerU command arguments
|
||||
// MinerU CLI: magic-pdf -p <input> -o <output_dir> -m auto
|
||||
const args = ["-p", filePath, "-o", mineruOutputDir, "-m", "auto"];
|
||||
/**
|
||||
* 執行 MinerU 並處理 vLLM 相容性問題
|
||||
* 如果 --table-mode 參數導致 vLLM 錯誤，會自動重試不帶此參數
|
||||
*/
|
||||
const runMinerU = (useTableMode: boolean): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Build MinerU command arguments
|
||||
// MinerU CLI: mineru -p <input> -o <output_dir> -m auto
|
||||
const args = ["-p", filePath, "-o", mineruOutputDir, "-m", "auto"];
|
||||
|
||||
// Add table mode option if md-i (render tables as images)
|
||||
if (convertTo === "md-i") {
|
||||
args.push("--table-mode", "image");
|
||||
} else {
|
||||
args.push("--table-mode", "markdown");
|
||||
}
|
||||
|
||||
execFile("mineru", args, async (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`mineru error: ${error}`);
|
||||
return;
|
||||
// 表格模式支援（可能與某些 vLLM 版本不相容）
|
||||
if (useTableMode) {
|
||||
if (convertTo === "md-i") {
|
||||
args.push("--table-mode", "image");
|
||||
} else {
|
||||
args.push("--table-mode", "markdown");
|
||||
}
|
||||
}
|
||||
|
||||
if (stdout) {
|
||||
console.log(`mineru stdout: ${stdout}`);
|
||||
}
|
||||
console.log(`[MinerU] Running: mineru ${args.join(" ")}`);
|
||||
|
||||
if (stderr) {
|
||||
console.error(`mineru stderr: ${stderr}`);
|
||||
}
|
||||
|
||||
try {
|
||||
// MinerU outputs to a subdirectory, find the actual output
|
||||
const mineruActualOutput = join(mineruOutputDir, "auto");
|
||||
|
||||
// Create .tar archive from the output directory (不使用壓縮)
|
||||
// 強制使用 .tar 格式，禁止 .tar.gz
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
console.log(`[MinerU] Target tar path: ${tarPath}`);
|
||||
|
||||
// Ensure the parent directory exists
|
||||
const tarDir = dirname(tarPath);
|
||||
if (!existsSync(tarDir)) {
|
||||
mkdirSync(tarDir, { recursive: true });
|
||||
execFile("mineru", args, (error, stdout, stderr) => {
|
||||
if (stdout) {
|
||||
console.log(`mineru stdout: ${stdout}`);
|
||||
}
|
||||
|
||||
// Use the actual MinerU output directory for archiving
|
||||
// MinerU 產生完整資料夾結構，全部封裝進 .tar
|
||||
const outputToArchive = existsSync(mineruActualOutput)
|
||||
? mineruActualOutput
|
||||
: mineruOutputDir;
|
||||
|
||||
console.log(`[MinerU] Archiving directory: ${outputToArchive}`);
|
||||
|
||||
// 列出要封裝的內容
|
||||
if (existsSync(outputToArchive)) {
|
||||
const contents = readdirSync(outputToArchive);
|
||||
console.log(`[MinerU] Archive contents: ${contents.join(", ")}`);
|
||||
if (stderr) {
|
||||
console.error(`mineru stderr: ${stderr}`);
|
||||
}
|
||||
|
||||
await createTarArchive(outputToArchive, tarPath, execFile);
|
||||
console.log(`[MinerU] Created archive: ${tarPath}`);
|
||||
if (error) {
|
||||
// 檢查是否為 vLLM table_mode 相容性錯誤
|
||||
const errorStr = String(error) + String(stderr);
|
||||
if (useTableMode && errorStr.includes("table_mode")) {
|
||||
console.warn(`[MinerU] ⚠️ table_mode 與 vLLM 不相容，重試不帶此參數...`);
|
||||
reject(new Error("RETRY_WITHOUT_TABLE_MODE"));
|
||||
} else {
|
||||
reject(new Error(`mineru error: ${error}`));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Clean up the temporary directory
|
||||
removeDir(mineruOutputDir);
|
||||
|
||||
resolve("Done");
|
||||
} catch (tarError) {
|
||||
reject(`Failed to create .tar archive: ${tarError}`);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// 嘗試執行 MinerU（自動處理 vLLM 相容性）
|
||||
try {
|
||||
await runMinerU(true);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === "RETRY_WITHOUT_TABLE_MODE") {
|
||||
// 清理輸出目錄並重試
|
||||
removeDir(mineruOutputDir);
|
||||
mkdirSync(mineruOutputDir, { recursive: true });
|
||||
await runMinerU(false);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 建立 .tar 封裝
|
||||
try {
|
||||
// MinerU outputs to a subdirectory, find the actual output
|
||||
const mineruActualOutput = join(mineruOutputDir, "auto");
|
||||
|
||||
// Create .tar archive from the output directory (不使用壓縮)
|
||||
// 強制使用 .tar 格式，禁止 .tar.gz
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
console.log(`[MinerU] Target tar path: ${tarPath}`);
|
||||
|
||||
// Ensure the parent directory exists
|
||||
const tarDir = dirname(tarPath);
|
||||
if (!existsSync(tarDir)) {
|
||||
mkdirSync(tarDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Use the actual MinerU output directory for archiving
|
||||
// MinerU 產生完整資料夾結構，全部封裝進 .tar
|
||||
const outputToArchive = existsSync(mineruActualOutput)
|
||||
? mineruActualOutput
|
||||
: mineruOutputDir;
|
||||
|
||||
console.log(`[MinerU] Archiving directory: ${outputToArchive}`);
|
||||
|
||||
// 列出要封裝的內容
|
||||
if (existsSync(outputToArchive)) {
|
||||
const contents = readdirSync(outputToArchive);
|
||||
console.log(`[MinerU] Archive contents: ${contents.join(", ")}`);
|
||||
}
|
||||
|
||||
await createTarArchive(outputToArchive, tarPath, execFile);
|
||||
console.log(`[MinerU] Created archive: ${tarPath}`);
|
||||
|
||||
// Clean up the temporary directory
|
||||
removeDir(mineruOutputDir);
|
||||
|
||||
return "Done";
|
||||
} catch (tarError) {
|
||||
throw new Error(`Failed to create .tar archive: ${tarError}`);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
import { execFile as execFileOriginal } from "node:child_process";
|
||||
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync, copyFileSync } from "node:fs";
|
||||
import { join, basename, dirname } from "node:path";
|
||||
import { ExecFileFn } from "./types";
|
||||
import { getArchiveFileName } from "../transfer";
|
||||
|
||||
// 翻譯服務優先順序（自動 fallback）
|
||||
const TRANSLATION_SERVICES = ["google", "bing"] as const;
|
||||
type TranslationService = (typeof TRANSLATION_SERVICES)[number];
|
||||
|
||||
/**
|
||||
* PDFMathTranslate Content Engine
|
||||
*
|
||||
|
|
@ -76,6 +79,31 @@ function extractTargetLanguage(convertTo: string): string {
|
|||
return match[1];
|
||||
}
|
||||
|
||||
/**
|
||||
* 標準化語言代碼為 pdf2zh 支援的格式
|
||||
* pdf2zh 使用 Google Translate 的語言代碼
|
||||
* @param lang 輸入語言代碼
|
||||
* @returns 標準化後的語言代碼
|
||||
*/
|
||||
function normalizeLanguageCode(lang: string): string {
|
||||
// pdf2zh / Google Translate 語言代碼映射
|
||||
const langMap: Record<string, string> = {
|
||||
// 繁體中文：pdf2zh 使用 "zh-TW" 或 "zh-Hant"
|
||||
"zh-tw": "zh-TW",
|
||||
"zh-hant": "zh-TW",
|
||||
"zht": "zh-TW",
|
||||
// 簡體中文
|
||||
"zh-cn": "zh-CN",
|
||||
"zh-hans": "zh-CN",
|
||||
"zhs": "zh-CN",
|
||||
"zh": "zh-CN",
|
||||
// 其他語言保持原樣
|
||||
};
|
||||
|
||||
const lowerLang = lang.toLowerCase();
|
||||
return langMap[lowerLang] || lang;
|
||||
}
|
||||
|
||||
/**
|
||||
* 檢查模型是否已預先下載
|
||||
* @returns 模型是否存在
|
||||
|
|
@ -98,13 +126,12 @@ function checkModelsExist(): boolean {
|
|||
function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputTar: string,
|
||||
execFile: ExecFileFn,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use tar command to create archive (without gzip compression)
|
||||
// tar -cf <output.tar> -C <sourceDir> .
|
||||
// 注意：使用 -cf 而非 -czf，避免 gzip 壓縮
|
||||
execFile("tar", ["-cf", outputTar, "-C", sourceDir, "."], (error, stdout, stderr) => {
|
||||
execFileOriginal("tar", ["-cf", outputTar, "-C", sourceDir, "."], (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`tar error: ${error}`);
|
||||
return;
|
||||
|
|
@ -139,24 +166,24 @@ function removeDir(dirPath: string): void {
|
|||
}
|
||||
|
||||
/**
|
||||
* 執行 pdf2zh 命令進行 PDF 翻譯
|
||||
* 執行 pdf2zh 命令進行 PDF 翻譯（單一服務）
|
||||
*
|
||||
* @param inputPath 輸入 PDF 路徑
|
||||
* @param outputDir 輸出目錄
|
||||
* @param targetLang 目標語言
|
||||
* @param execFile 執行函數
|
||||
* @param service 翻譯服務（google, bing）
|
||||
*/
|
||||
function runPdf2zh(
|
||||
function runPdf2zhWithService(
|
||||
inputPath: string,
|
||||
outputDir: string,
|
||||
targetLang: string,
|
||||
execFile: ExecFileFn,
|
||||
service: TranslationService,
|
||||
): Promise<{ monoPath: string; dualPath: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// pdf2zh CLI 參數：
|
||||
// -lo <lang>: 目標語言
|
||||
// -o <dir>: 輸出目錄
|
||||
// -s google: 使用 Google 翻譯（預設，免費）
|
||||
// -s <service>: 翻譯服務（google, bing, deepl, ollama 等）
|
||||
//
|
||||
// 輸出檔案：
|
||||
// - <filename>-mono.pdf: 純翻譯版本
|
||||
|
|
@ -169,7 +196,7 @@ function runPdf2zh(
|
|||
"-o",
|
||||
outputDir,
|
||||
"-s",
|
||||
process.env.PDFMATHTRANSLATE_SERVICE || "google",
|
||||
service,
|
||||
];
|
||||
|
||||
// 如果設定了自訂模型路徑，使用 --onnx 參數
|
||||
|
|
@ -180,11 +207,12 @@ function runPdf2zh(
|
|||
}
|
||||
}
|
||||
|
||||
console.log(`[PDFMathTranslate] Running: pdf2zh ${args.join(" ")}`);
|
||||
console.log(`[PDFMathTranslate] Running: pdf2zh ${args.join(" ")} (service: ${service})`);
|
||||
|
||||
execFile("pdf2zh", args, (error, stdout, stderr) => {
|
||||
// 使用原生 execFileOriginal 以支援 timeout 選項
|
||||
execFileOriginal("pdf2zh", args, { timeout: 300000, maxBuffer: 50 * 1024 * 1024 }, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`pdf2zh error: ${error}\nstderr: ${stderr}`);
|
||||
reject(new Error(`pdf2zh error (${service}): ${error}\nstderr: ${stderr}`));
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -209,7 +237,7 @@ function runPdf2zh(
|
|||
console.log(`[PDFMathTranslate] Found PDF files: ${pdfFiles.join(", ")}`);
|
||||
|
||||
if (pdfFiles.length === 0) {
|
||||
reject(`No output PDF files found in ${outputDir}`);
|
||||
reject(new Error(`No output PDF files found in ${outputDir} (service: ${service})`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -219,6 +247,62 @@ function runPdf2zh(
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 執行 pdf2zh 命令進行 PDF 翻譯（自動 fallback）
|
||||
*
|
||||
* 嘗試順序：
|
||||
* 1. 環境變數 PDFMATHTRANSLATE_SERVICE（如果設定）
|
||||
* 2. Google Translate（免費，需要網路）
|
||||
* 3. Bing Translate（免費備援）
|
||||
*
|
||||
* @param inputPath 輸入 PDF 路徑
|
||||
* @param outputDir 輸出目錄
|
||||
* @param targetLang 目標語言
|
||||
*/
|
||||
async function runPdf2zh(
|
||||
inputPath: string,
|
||||
outputDir: string,
|
||||
targetLang: string,
|
||||
): Promise<{ monoPath: string; dualPath: string }> {
|
||||
// 如果使用者指定了服務，只用那個服務
|
||||
const envService = process.env.PDFMATHTRANSLATE_SERVICE;
|
||||
if (envService) {
|
||||
console.log(`[PDFMathTranslate] Using user-specified service: ${envService}`);
|
||||
return runPdf2zhWithService(inputPath, outputDir, targetLang, envService as TranslationService);
|
||||
}
|
||||
|
||||
// 自動 fallback：依序嘗試各個服務
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const service of TRANSLATION_SERVICES) {
|
||||
try {
|
||||
console.log(`[PDFMathTranslate] Trying translation service: ${service}`);
|
||||
const result = await runPdf2zhWithService(inputPath, outputDir, targetLang, service);
|
||||
console.log(`[PDFMathTranslate] ✅ Success with service: ${service}`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
console.warn(`[PDFMathTranslate] ⚠️ Service ${service} failed: ${errorMsg}`);
|
||||
errors.push(`${service}: ${errorMsg}`);
|
||||
|
||||
// 清理失敗的輸出檔案，準備下一次嘗試
|
||||
try {
|
||||
const files = readdirSync(outputDir);
|
||||
for (const file of files) {
|
||||
if (file.endsWith(".pdf") && file !== basename(inputPath)) {
|
||||
unlinkSync(join(outputDir, file));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 忽略清理錯誤
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 所有服務都失敗
|
||||
throw new Error(`All translation services failed:\n${errors.join("\n")}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* 主要轉換函數
|
||||
*
|
||||
|
|
@ -227,7 +311,6 @@ function runPdf2zh(
|
|||
* @param convertTo 目標格式（如 "pdf-zh"）
|
||||
* @param targetPath 輸出路徑
|
||||
* @param options 額外選項
|
||||
* @param execFile 執行函數覆寫
|
||||
*/
|
||||
export async function convert(
|
||||
filePath: string,
|
||||
|
|
@ -235,15 +318,15 @@ export async function convert(
|
|||
convertTo: string,
|
||||
targetPath: string,
|
||||
_options?: unknown,
|
||||
execFile: ExecFileFn = execFileOriginal,
|
||||
): Promise<string> {
|
||||
try {
|
||||
// 1. 檢查模型（警告但不阻止，因為 pdf2zh 可能會自動下載）
|
||||
checkModelsExist();
|
||||
|
||||
// 2. 提取目標語言
|
||||
const targetLang = extractTargetLanguage(convertTo);
|
||||
console.log(`[PDFMathTranslate] Translating to: ${targetLang}`);
|
||||
// 2. 提取並標準化目標語言
|
||||
const rawLang = extractTargetLanguage(convertTo);
|
||||
const targetLang = normalizeLanguageCode(rawLang);
|
||||
console.log(`[PDFMathTranslate] Translating to: ${targetLang} (raw: ${rawLang})`);
|
||||
|
||||
// 3. 建立臨時輸出目錄
|
||||
const outputDir = dirname(targetPath);
|
||||
|
|
@ -259,7 +342,7 @@ export async function convert(
|
|||
mkdirSync(archiveDir, { recursive: true });
|
||||
|
||||
// 5. 執行 pdf2zh 翻譯
|
||||
const { monoPath, dualPath } = await runPdf2zh(filePath, tempDir, targetLang, execFile);
|
||||
const { monoPath, dualPath } = await runPdf2zh(filePath, tempDir, targetLang);
|
||||
|
||||
// 6. 複製翻譯後的檔案到封裝目錄
|
||||
// PDFMathTranslate 輸出：
|
||||
|
|
@ -331,7 +414,7 @@ export async function convert(
|
|||
mkdirSync(tarDir, { recursive: true });
|
||||
}
|
||||
|
||||
await createTarArchive(archiveDir, tarPath, execFile);
|
||||
await createTarArchive(archiveDir, tarPath);
|
||||
console.log(`[PDFMathTranslate] Created archive: ${tarPath}`);
|
||||
|
||||
// 9. 清理臨時目錄
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue