refactor(docker): 完整重構 Dockerfile v0.1.16
BREAKING CHANGE: Dockerfile 結構重新設計 ## 重構內容 ### Multi-Stage Build 結構 - Stage 1 [base]: Bun runtime 基礎 - Stage 2 [install]: Node 依賴安裝 - Stage 3 [prerelease]: 應用程式建構 - Stage 4 [system-tools]: APT 系統工具（拆分為 14 個獨立 RUN） - Stage 5 [fonts]: 字型安裝 - Stage 6 [python-tools]: Python CLI 工具（拆分為 9 個獨立 RUN） - Stage 7 [models]: 模型下載 - Stage 8 [release]: 最終 Image ### 新增 Scripts - install-system-tools.sh: 系統工具安裝 - install-fonts.sh: 字型安裝 - install-python-tools.sh: Python 工具安裝 - download-models.sh: 模型下載 - verify-installation.sh: 完整安裝驗證 ### 刪除舊檔案 - Dockerfile.backup - Dockerfile.offline - Dockerfile.old ### 路徑更新 - MinerU 模型: /opt/convertx/models/mineru/PDF-Extract-Kit-1.0 - BabelDOC 快取: /root/.cache/babeldoc ### Multi-Arch 支援 - linux/amd64: 功能完整 - linux/arm64: 安全降級（MinerU/resvg 跳過） Closes #docker-refactor
This commit is contained in:
Your Name
parent
e43649bc96
commit
55b0f519ab
10 changed files with 1797 additions and 2008 deletions
306
scripts/download-models.sh
Normal file
306
scripts/download-models.sh
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
#!/usr/bin/env bash
|
||||
# ==============================================================================
|
||||
# ConvertX-CN 模型下載腳本
|
||||
# ==============================================================================
|
||||
# 用途：下載所有 AI 模型（MinerU, BabelDOC, PDFMathTranslate）
|
||||
# 執行環境：Docker build stage (models-download)
|
||||
#
|
||||
# ⚠️ 重要：此腳本在 build time 執行，下載所有模型到固定目錄
|
||||
# Runtime 完全離線，不得有任何下載行為
|
||||
# ==============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# 顏色輸出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${BLUE}📥 [INFO]${NC} $1"; }
|
||||
log_success() { echo -e "${GREEN}✅ [SUCCESS]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}⚠️ [WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}❌ [ERROR]${NC} $1"; }
|
||||
|
||||
# 架構檢測
|
||||
ARCH=$(uname -m)
|
||||
log_info "檢測架構: ${ARCH}"
|
||||
|
||||
# ==============================================================================
|
||||
# 目錄定義
|
||||
# ==============================================================================
|
||||
MODELS_BASE_DIR="/opt/convertx/models"
|
||||
MINERU_MODELS_DIR="${MODELS_BASE_DIR}/mineru"
|
||||
BABELDOC_CACHE_DIR="/root/.cache/babeldoc"
|
||||
BABELDOC_MODELS_DIR="${BABELDOC_CACHE_DIR}/models"
|
||||
BABELDOC_FONTS_DIR="${BABELDOC_CACHE_DIR}/fonts"
|
||||
BABELDOC_CMAP_DIR="${BABELDOC_CACHE_DIR}/cmap"
|
||||
BABELDOC_TIKTOKEN_DIR="${BABELDOC_CACHE_DIR}/tiktoken"
|
||||
|
||||
# ==============================================================================
|
||||
# 創建目錄結構
|
||||
# ==============================================================================
|
||||
create_directories() {
|
||||
log_info "創建模型目錄結構..."
|
||||
|
||||
mkdir -p "${MINERU_MODELS_DIR}"
|
||||
mkdir -p "${BABELDOC_MODELS_DIR}"
|
||||
mkdir -p "${BABELDOC_FONTS_DIR}"
|
||||
mkdir -p "${BABELDOC_CMAP_DIR}"
|
||||
mkdir -p "${BABELDOC_TIKTOKEN_DIR}"
|
||||
|
||||
log_success "目錄結構創建完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 下載 MinerU Pipeline 模型
|
||||
# ==============================================================================
|
||||
download_mineru_models() {
|
||||
log_info "下載 MinerU Pipeline 模型..."
|
||||
|
||||
if [ "${ARCH}" = "aarch64" ]; then
|
||||
log_warn "ARM64：跳過 MinerU 模型下載（不支援）"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 << 'PYTHON'
|
||||
from huggingface_hub import snapshot_download
|
||||
import os
|
||||
|
||||
models_dir = os.environ.get('MINERU_MODELS_DIR', '/opt/convertx/models/mineru')
|
||||
print(f'下載 PDF-Extract-Kit-1.0 到 {models_dir}...')
|
||||
|
||||
try:
|
||||
snapshot_download(
|
||||
repo_id='opendatalab/PDF-Extract-Kit-1.0',
|
||||
local_dir=f'{models_dir}/PDF-Extract-Kit-1.0',
|
||||
local_dir_use_symlinks=False,
|
||||
resume_download=True
|
||||
)
|
||||
print('✅ PDF-Extract-Kit-1.0 下載完成')
|
||||
except Exception as e:
|
||||
print(f'❌ 下載失敗: {e}')
|
||||
raise
|
||||
PYTHON
|
||||
|
||||
# 顯示模型大小
|
||||
log_info "MinerU 模型目錄內容:"
|
||||
du -sh "${MINERU_MODELS_DIR}/PDF-Extract-Kit-1.0" 2>/dev/null || true
|
||||
ls -la "${MINERU_MODELS_DIR}/" 2>/dev/null || true
|
||||
|
||||
log_success "MinerU 模型下載完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 產生 MinerU 配置檔
|
||||
# ==============================================================================
|
||||
generate_mineru_config() {
|
||||
log_info "產生 MinerU 配置檔..."
|
||||
|
||||
if [ "${ARCH}" = "aarch64" ]; then
|
||||
log_warn "ARM64：跳過 MinerU 配置"
|
||||
return 0
|
||||
fi
|
||||
|
||||
python3 << 'PYTHON'
|
||||
import json
|
||||
import os
|
||||
|
||||
mineru_models_dir = os.environ.get('MINERU_MODELS_DIR', '/opt/convertx/models/mineru')
|
||||
|
||||
config = {
|
||||
'models-dir': {
|
||||
'pipeline': f'{mineru_models_dir}/PDF-Extract-Kit-1.0',
|
||||
'vlm': ''
|
||||
},
|
||||
'model-source': 'local',
|
||||
'latex-delimiter-config': {
|
||||
'display': {'left': '@@', 'right': '@@'},
|
||||
'inline': {'left': '@', 'right': '@'}
|
||||
}
|
||||
}
|
||||
|
||||
# 寫入多個位置
|
||||
config_paths = [
|
||||
'/opt/convertx/mineru.json',
|
||||
'/root/mineru.json'
|
||||
]
|
||||
|
||||
for path in config_paths:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
print(f'✅ 已產生: {path}')
|
||||
|
||||
print(json.dumps(config, indent=2))
|
||||
PYTHON
|
||||
|
||||
log_success "MinerU 配置檔產生完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# BabelDOC warmup（下載所有資源）
|
||||
# ==============================================================================
|
||||
babeldoc_warmup() {
|
||||
log_info "執行 BabelDOC warmup..."
|
||||
|
||||
export BABELDOC_CACHE_PATH="${BABELDOC_CACHE_DIR}"
|
||||
|
||||
if command -v babeldoc >/dev/null 2>&1; then
|
||||
babeldoc --warmup 2>&1 || {
|
||||
log_warn "BabelDOC warmup 可能有警告，但繼續"
|
||||
}
|
||||
|
||||
log_info "BabelDOC 快取內容:"
|
||||
ls -la "${BABELDOC_CACHE_DIR}/" 2>/dev/null || true
|
||||
du -sh "${BABELDOC_CACHE_DIR}/" 2>/dev/null || true
|
||||
else
|
||||
log_warn "babeldoc 不可用，跳過 warmup"
|
||||
fi
|
||||
|
||||
log_success "BabelDOC warmup 完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 下載 tiktoken 編碼
|
||||
# ==============================================================================
|
||||
download_tiktoken() {
|
||||
log_info "下載 tiktoken 編碼..."
|
||||
|
||||
python3 << 'PYTHON'
|
||||
try:
|
||||
import tiktoken
|
||||
|
||||
encodings = ['cl100k_base', 'p50k_base', 'r50k_base']
|
||||
|
||||
for enc_name in encodings:
|
||||
try:
|
||||
enc = tiktoken.get_encoding(enc_name)
|
||||
print(f'✅ tiktoken {enc_name} 已下載')
|
||||
except Exception as e:
|
||||
print(f'⚠️ tiktoken {enc_name} 下載失敗: {e}')
|
||||
|
||||
except ImportError:
|
||||
print('⚠️ tiktoken 未安裝，跳過')
|
||||
PYTHON
|
||||
|
||||
log_success "tiktoken 編碼下載完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 複製預下載的 ONNX 模型（從 COPY 複製）
|
||||
# ==============================================================================
|
||||
setup_onnx_models() {
|
||||
log_info "設定 ONNX 模型..."
|
||||
|
||||
# 如果有預下載的模型，複製到正確位置
|
||||
local ONNX_MODEL="doclayout_yolo_docstructbench_imgsz1024.onnx"
|
||||
local SRC_PATHS=(
|
||||
"/app/models/${ONNX_MODEL}"
|
||||
"/tmp/models/${ONNX_MODEL}"
|
||||
)
|
||||
|
||||
for src in "${SRC_PATHS[@]}"; do
|
||||
if [ -f "${src}" ]; then
|
||||
cp "${src}" "${BABELDOC_MODELS_DIR}/"
|
||||
log_success "已複製 ONNX 模型: ${src} -> ${BABELDOC_MODELS_DIR}/"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
log_warn "未找到預下載的 ONNX 模型，將依賴 babeldoc --warmup"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 驗證下載結果
|
||||
# ==============================================================================
|
||||
verify_downloads() {
|
||||
log_info "驗證模型下載..."
|
||||
|
||||
echo ""
|
||||
echo "模型目錄統計:"
|
||||
echo "=============================================="
|
||||
|
||||
# MinerU
|
||||
if [ "${ARCH}" != "aarch64" ]; then
|
||||
if [ -d "${MINERU_MODELS_DIR}/PDF-Extract-Kit-1.0" ]; then
|
||||
local size
|
||||
size=$(du -sh "${MINERU_MODELS_DIR}/PDF-Extract-Kit-1.0" 2>/dev/null | awk '{print $1}')
|
||||
echo " ✅ MinerU Pipeline: ${size}"
|
||||
else
|
||||
echo " ❌ MinerU Pipeline: 不存在"
|
||||
fi
|
||||
|
||||
if [ -f "/root/mineru.json" ]; then
|
||||
echo " ✅ MinerU 配置檔: 已產生"
|
||||
else
|
||||
echo " ❌ MinerU 配置檔: 不存在"
|
||||
fi
|
||||
else
|
||||
echo " ⚠️ MinerU: ARM64 跳過"
|
||||
fi
|
||||
|
||||
# BabelDOC
|
||||
if [ -d "${BABELDOC_CACHE_DIR}" ]; then
|
||||
local size
|
||||
size=$(du -sh "${BABELDOC_CACHE_DIR}" 2>/dev/null | awk '{print $1}')
|
||||
echo " ✅ BabelDOC 快取: ${size}"
|
||||
else
|
||||
echo " ⚠️ BabelDOC 快取: 不存在"
|
||||
fi
|
||||
|
||||
# ONNX 模型
|
||||
if [ -f "${BABELDOC_MODELS_DIR}/doclayout_yolo_docstructbench_imgsz1024.onnx" ]; then
|
||||
local size
|
||||
size=$(ls -lh "${BABELDOC_MODELS_DIR}/doclayout_yolo_docstructbench_imgsz1024.onnx" | awk '{print $5}')
|
||||
echo " ✅ DocLayout-YOLO ONNX: ${size}"
|
||||
else
|
||||
echo " ⚠️ DocLayout-YOLO ONNX: 不存在"
|
||||
fi
|
||||
|
||||
echo "=============================================="
|
||||
log_success "驗證完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 清理下載快取
|
||||
# ==============================================================================
|
||||
cleanup_cache() {
|
||||
log_info "清理下載快取..."
|
||||
|
||||
rm -rf /tmp/hf_download_cache 2>/dev/null || true
|
||||
rm -rf /root/.cache/huggingface 2>/dev/null || true
|
||||
rm -rf /root/.cache/pip 2>/dev/null || true
|
||||
rm -rf /root/.cache/uv 2>/dev/null || true
|
||||
|
||||
# 清理 Python cache
|
||||
find /usr -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
log_success "快取清理完成"
|
||||
}
|
||||
|
||||
# ==============================================================================
|
||||
# 主程式
|
||||
# ==============================================================================
|
||||
main() {
|
||||
log_info "=========================================="
|
||||
log_info "開始下載模型 (${ARCH})"
|
||||
log_info "=========================================="
|
||||
|
||||
create_directories
|
||||
download_mineru_models
|
||||
generate_mineru_config
|
||||
setup_onnx_models
|
||||
babeldoc_warmup
|
||||
download_tiktoken
|
||||
verify_downloads
|
||||
cleanup_cache
|
||||
|
||||
log_success "=========================================="
|
||||
log_success "所有模型下載完成！"
|
||||
log_success "=========================================="
|
||||
}
|
||||
|
||||
# 執行
|
||||
main "$@"
|
||||
Loading…
Add table
Add a link
Reference in a new issue