fix(docker): replace python3 -c with heredoc scripts to fix IndentationError
This commit is contained in:
parent
a9b1fc5699
commit
906f112999
4 changed files with 117 additions and 63 deletions
36
scripts/download-mineru-models.sh
Normal file
36
scripts/download-mineru-models.sh
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# MinerU 模型下載腳本
|
||||
# 用於 Docker build 階段下載 PDF-Extract-Kit-1.0
|
||||
# ==============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
ARCH=$(uname -m)
|
||||
MODELS_DIR="${MINERU_MODELS_DIR:-/opt/convertx/models/mineru}"
|
||||
|
||||
if [ "$ARCH" = "aarch64" ]; then
|
||||
echo "⚠️ ARM64:跳過 MinerU 模型下載"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "📦 下載 MinerU PDF-Extract-Kit-1.0 到 ${MODELS_DIR}..."
|
||||
|
||||
python3 <<'PYTHON'
|
||||
from huggingface_hub import snapshot_download
|
||||
import os
|
||||
|
||||
models_dir = os.environ.get('MINERU_MODELS_DIR', '/opt/convertx/models/mineru')
|
||||
print(f'下載 PDF-Extract-Kit-1.0 到 {models_dir}...')
|
||||
|
||||
snapshot_download(
|
||||
repo_id='opendatalab/PDF-Extract-Kit-1.0',
|
||||
local_dir=f'{models_dir}/PDF-Extract-Kit-1.0',
|
||||
local_dir_use_symlinks=False,
|
||||
resume_download=True
|
||||
)
|
||||
|
||||
print('✅ PDF-Extract-Kit-1.0 下載完成')
|
||||
PYTHON
|
||||
|
||||
echo "✅ MinerU 模型下載完成"
|
||||
24
scripts/download-tiktoken.sh
Normal file
24
scripts/download-tiktoken.sh
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# tiktoken 編碼下載腳本
|
||||
# 用於 Docker build 階段預下載 tiktoken 編碼
|
||||
# ==============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
echo "📦 下載 tiktoken 編碼..."
|
||||
|
||||
python3 <<'PYTHON'
|
||||
try:
|
||||
import tiktoken
|
||||
for enc_name in ['cl100k_base', 'p50k_base', 'r50k_base']:
|
||||
try:
|
||||
enc = tiktoken.get_encoding(enc_name)
|
||||
print(f'✅ tiktoken {enc_name} 已下載')
|
||||
except Exception as e:
|
||||
print(f'⚠️ tiktoken {enc_name} 下載失敗: {e}')
|
||||
except ImportError:
|
||||
print('⚠️ tiktoken 未安裝,跳過')
|
||||
PYTHON
|
||||
|
||||
echo "✅ tiktoken 編碼下載完成"
|
||||
50
scripts/generate-mineru-config.sh
Normal file
50
scripts/generate-mineru-config.sh
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# MinerU 配置檔產生腳本
|
||||
# 用於 Docker build 階段產生 mineru.json
|
||||
# ==============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
ARCH=$(uname -m)
|
||||
MODELS_DIR="${MINERU_MODELS_DIR:-/opt/convertx/models/mineru}"
|
||||
|
||||
mkdir -p /opt/convertx
|
||||
|
||||
if [ "$ARCH" = "aarch64" ]; then
|
||||
echo '{"models-dir":{"pipeline":"","vlm":""},"model-source":"local","note":"ARM64 - MinerU not supported"}' > /opt/convertx/mineru.json
|
||||
cp /opt/convertx/mineru.json /root/mineru.json
|
||||
echo "⚠️ ARM64:產生空的 mineru.json"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 <<'PYTHON'
|
||||
import json
|
||||
import os
|
||||
|
||||
mineru_models_dir = os.environ.get('MINERU_MODELS_DIR', '/opt/convertx/models/mineru')
|
||||
|
||||
config = {
|
||||
'models-dir': {
|
||||
'pipeline': f'{mineru_models_dir}/PDF-Extract-Kit-1.0',
|
||||
'vlm': ''
|
||||
},
|
||||
'model-source': 'local',
|
||||
'latex-delimiter-config': {
|
||||
'display': {'left': '@@', 'right': '@@'},
|
||||
'inline': {'left': '@', 'right': '@'}
|
||||
}
|
||||
}
|
||||
|
||||
os.makedirs('/opt/convertx', exist_ok=True)
|
||||
|
||||
with open('/opt/convertx/mineru.json', 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
with open('/root/mineru.json', 'w') as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
print('✅ mineru.json 已產生')
|
||||
PYTHON
|
||||
|
||||
echo "✅ MinerU 配置檔產生完成"
|
||||
Loading…
Add table
Add a link
Reference in a new issue