fix(docker): 修復三個關鍵問題
1. pykcs11 編譯失敗:安裝 build-essential, swig, libpcsclite-dev 2. MinerU 模型 cache 問題:模型移動到 /opt/mineru/models/ 固定目錄 - 不再依賴 HF cache(可安全刪除) - mineru.json 指向固定目錄 3. 修復驗證邏輯縮排錯誤 CACHE_BUST: 5
This commit is contained in:
parent
0b1ecb7b40
commit
b23d42a7b1
1 changed files with 93 additions and 69 deletions
54
Dockerfile
54
Dockerfile
|
|
@ -401,17 +401,28 @@ ENV PDF_SIGN_CONTACT="convertx-cn@localhost"
|
||||||
# 當 CACHE_BUST 改變時,後續所有層都會重新執行
|
# 當 CACHE_BUST 改變時,後續所有層都會重新執行
|
||||||
# 這確保模型下載不會被損壞的 cache 跳過
|
# 這確保模型下載不會被損壞的 cache 跳過
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
ARG CACHE_BUST=4
|
ARG CACHE_BUST=5
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# 階段 12A:安裝 huggingface_hub + endesive
|
# 階段 12A:安裝 huggingface_hub + endesive
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
# ⚠️ endesive 依賴 pykcs11,需要 C++ 編譯器和 SWIG
|
||||||
RUN echo "Cache bust: ${CACHE_BUST}" && \
|
RUN echo "Cache bust: ${CACHE_BUST}" && \
|
||||||
set -eu && \
|
set -eu && \
|
||||||
echo "===========================================================" && \
|
echo "===========================================================" && \
|
||||||
echo "📦 [1/8] 安裝 huggingface_hub + endesive(PDF 簽章)..." && \
|
echo "📦 [1/8] 安裝 huggingface_hub + endesive(PDF 簽章)..." && \
|
||||||
echo "===========================================================" && \
|
echo "===========================================================" && \
|
||||||
|
# 安裝 pykcs11 編譯依賴
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
swig \
|
||||||
|
libpcsclite-dev && \
|
||||||
pip3 install --no-cache-dir --break-system-packages huggingface_hub endesive && \
|
pip3 install --no-cache-dir --break-system-packages huggingface_hub endesive && \
|
||||||
|
# 清理編譯依賴(保留 libpcsclite-dev runtime 需要)
|
||||||
|
apt-get remove -y build-essential swig && \
|
||||||
|
apt-get autoremove -y && \
|
||||||
|
rm -rf /var/lib/apt/lists/* && \
|
||||||
echo "✅ huggingface_hub + endesive 安裝完成"
|
echo "✅ huggingface_hub + endesive 安裝完成"
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
|
@ -533,24 +544,41 @@ RUN set -eu && \
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# 階段 12G:MinerU 模型下載
|
# 階段 12G:MinerU 模型下載
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
# ⚠️ 關鍵:模型必須移動到固定目錄,不能依賴 HF cache
|
||||||
|
# 否則後續清理 HF cache 會導致 runtime 炸掉
|
||||||
RUN set -eu && \
|
RUN set -eu && \
|
||||||
echo "===========================================================" && \
|
echo "===========================================================" && \
|
||||||
echo "📥 [7/8] 下載 MinerU Pipeline 模型..." && \
|
echo "📥 [7/8] 下載 MinerU Pipeline 模型..." && \
|
||||||
echo "===========================================================" && \
|
echo "===========================================================" && \
|
||||||
ARCH=$(uname -m) && \
|
ARCH=$(uname -m) && \
|
||||||
|
MINERU_MODELS_DIR="/opt/mineru/models" && \
|
||||||
if [ "$ARCH" = "aarch64" ]; then \
|
if [ "$ARCH" = "aarch64" ]; then \
|
||||||
echo "⚠️ ARM64 架構:MinerU 不支援,跳過模型下載"; \
|
echo "⚠️ ARM64 架構:MinerU 不支援,跳過模型下載"; \
|
||||||
else \
|
else \
|
||||||
|
mkdir -p "$MINERU_MODELS_DIR" && \
|
||||||
|
echo "目標目錄:$MINERU_MODELS_DIR" && \
|
||||||
|
# 嘗試使用 mineru-models-download CLI
|
||||||
if command -v mineru-models-download >/dev/null 2>&1; then \
|
if command -v mineru-models-download >/dev/null 2>&1; then \
|
||||||
echo "使用 mineru-models-download CLI..." && \
|
echo "使用 mineru-models-download CLI..." && \
|
||||||
(mineru-models-download -s huggingface -m pipeline 2>&1 || echo "⚠️ mineru-models-download 失敗,嘗試手動下載...") && \
|
mineru-models-download -s huggingface -m pipeline && \
|
||||||
cat /root/mineru.json 2>/dev/null || echo "(mineru.json 未生成)"; \
|
# mineru-models-download 會建立 /root/mineru.json 並下載到 HF cache
|
||||||
|
# 需要讀取 mineru.json 找到模型位置並移動
|
||||||
|
if [ -f /root/mineru.json ]; then \
|
||||||
|
DOWNLOADED_DIR=$(python3 -c "import json; f=open('/root/mineru.json'); d=json.load(f); print(d.get('models-dir',{}).get('pipeline',''))" 2>/dev/null || echo "") && \
|
||||||
|
if [ -n "$DOWNLOADED_DIR" ] && [ -d "$DOWNLOADED_DIR" ]; then \
|
||||||
|
echo "移動模型從 $DOWNLOADED_DIR 到 $MINERU_MODELS_DIR/PDF-Extract-Kit-1.0" && \
|
||||||
|
mv "$DOWNLOADED_DIR" "$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0"; \
|
||||||
|
fi; \
|
||||||
|
fi; \
|
||||||
else \
|
else \
|
||||||
echo "mineru-models-download 不可用,使用顯式 HuggingFace 下載..." && \
|
echo "mineru-models-download 不可用,使用顯式 HuggingFace 下載..." && \
|
||||||
mkdir -p /root/.cache/mineru/models && \
|
python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='opendatalab/PDF-Extract-Kit-1.0', local_dir='$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0', local_dir_use_symlinks=False); print('PDF-Extract-Kit-1.0 downloaded')"; \
|
||||||
(python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='opendatalab/PDF-Extract-Kit-1.0', local_dir='/root/.cache/mineru/models/PDF-Extract-Kit-1.0', local_dir_use_symlinks=False); print('PDF-Extract-Kit-1.0 downloaded')" || echo "⚠️ MinerU model download failed") && \
|
fi && \
|
||||||
python3 -c "import json; config={'models-dir':{'pipeline':'/root/.cache/mineru/models/PDF-Extract-Kit-1.0','vlm':''},'model-source':'local','latex-delimiter-config':{'display':{'left':'@@','right':'@@'},'inline':{'left':'@','right':'@'}}}; f=open('/root/mineru.json','w'); json.dump(config,f,indent=2); f.close(); print('mineru.json generated')"; \
|
# 產生/更新 mineru.json 指向固定目錄
|
||||||
fi; \
|
python3 -c "import json; config={'models-dir':{'pipeline':'$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0','vlm':''},'model-source':'local','latex-delimiter-config':{'display':{'left':'@@','right':'@@'},'inline':{'left':'@','right':'@'}}}; f=open('/root/mineru.json','w'); json.dump(config,f,indent=2); f.close(); print('mineru.json updated')" && \
|
||||||
|
echo "驗證模型目錄:" && \
|
||||||
|
ls -la "$MINERU_MODELS_DIR/" && \
|
||||||
|
du -sh "$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0" 2>/dev/null || echo "(無法計算大小)"; \
|
||||||
fi && \
|
fi && \
|
||||||
echo "✅ MinerU 模型下載步驟完成"
|
echo "✅ MinerU 模型下載步驟完成"
|
||||||
|
|
||||||
|
|
@ -588,21 +616,17 @@ RUN set -eu && \
|
||||||
echo " ⏭️ 跳過驗證(字型將透過 COPY fonts/ 指令複製)" && \
|
echo " ⏭️ 跳過驗證(字型將透過 COPY fonts/ 指令複製)" && \
|
||||||
\
|
\
|
||||||
echo "🔍 驗證 MinerU 安裝..." && \
|
echo "🔍 驗證 MinerU 安裝..." && \
|
||||||
|
MINERU_MODELS_DIR="/opt/mineru/models" && \
|
||||||
if [ "$ARCH" = "aarch64" ]; then \
|
if [ "$ARCH" = "aarch64" ]; then \
|
||||||
echo " ⚠️ ARM64 架構:跳過 MinerU 驗證"; \
|
echo " ⚠️ ARM64 架構:跳過 MinerU 驗證"; \
|
||||||
elif command -v mineru >/dev/null 2>&1; then \
|
elif command -v mineru >/dev/null 2>&1; then \
|
||||||
echo " ✅ MinerU 已安裝: $(command -v mineru)" && \
|
echo " ✅ MinerU 已安裝: $(command -v mineru)" && \
|
||||||
if [ -f /root/mineru.json ]; then \
|
if [ -d "$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0" ]; then \
|
||||||
MINERU_DIR=$(python3 -c "import json; f=open('/root/mineru.json'); d=json.load(f); print(d.get('models-dir',{}).get('pipeline',''))" 2>/dev/null || echo "") && \
|
MINERU_SIZE=$(du -sb "$MINERU_MODELS_DIR/PDF-Extract-Kit-1.0" 2>/dev/null | cut -f1 || echo "0") && \
|
||||||
if [ -n "$MINERU_DIR" ] && [ -d "$MINERU_DIR" ]; then \
|
echo " ✅ MinerU 模型存在於固定目錄 ($((MINERU_SIZE/1024/1024)) MB)"; \
|
||||||
MINERU_SIZE=$(du -sb "$MINERU_DIR" 2>/dev/null | cut -f1 || echo "0") && \
|
|
||||||
echo " ✅ MinerU 模型存在 ($((MINERU_SIZE/1024/1024)) MB)"; \
|
|
||||||
else \
|
else \
|
||||||
echo " ⚠️ MinerU 模型目錄不存在(模型將在首次使用時下載)"; \
|
echo " ⚠️ MinerU 模型目錄不存在(模型將在首次使用時下載)"; \
|
||||||
fi; \
|
fi; \
|
||||||
else \
|
|
||||||
echo " ⚠️ mineru.json 不存在(模型將在首次使用時下載)"; \
|
|
||||||
fi; \
|
|
||||||
else \
|
else \
|
||||||
echo " ❌ MinerU 未安裝(amd64 必須安裝)" && \
|
echo " ❌ MinerU 未安裝(amd64 必須安裝)" && \
|
||||||
VALIDATION_FAILED=1; \
|
VALIDATION_FAILED=1; \
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue