feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小) - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器(支援直傳與 chunk) - downloadManager.ts: 後端下載管理器(支援直傳與 chunk) - archiveManager.ts: 封裝管理器(僅允許 .tar) - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk) - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB:直接傳輸 - 檔案 >10MB:使用 5MB chunks 分段傳輸 - 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip) - 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
46
README.md
46
README.md
|
|
@ -157,15 +157,15 @@ docker compose up -d
|
|||
|
||||
## 支援格式
|
||||
|
||||
| 轉換器 | 用途 | 格式數 |
|
||||
| ----------- | ------ | ------ |
|
||||
| FFmpeg | 影音 | 400+ |
|
||||
| ImageMagick | 圖片 | 200+ |
|
||||
| LibreOffice | 文件 | 60+ |
|
||||
| Pandoc | 文件 | 100+ |
|
||||
| Calibre | 電子書 | 40+ |
|
||||
| Inkscape | 向量圖 | 20+ |
|
||||
| MinerU | 文件→MD | 2 |
|
||||
| 轉換器 | 用途 | 格式數 |
|
||||
| ----------- | ------- | ------ |
|
||||
| FFmpeg | 影音 | 400+ |
|
||||
| ImageMagick | 圖片 | 200+ |
|
||||
| LibreOffice | 文件 | 60+ |
|
||||
| Pandoc | 文件 | 100+ |
|
||||
| Calibre | 電子書 | 40+ |
|
||||
| Inkscape | 向量圖 | 20+ |
|
||||
| MinerU | 文件→MD | 2 |
|
||||
|
||||
完整列表 → [docs/converters.md](docs/converters.md)
|
||||
|
||||
|
|
@ -175,10 +175,32 @@ docker compose up -d
|
|||
|
||||
ConvertX 內建文件轉換引擎。
|
||||
|
||||
- md-t
|
||||
- md-i
|
||||
- md-t(表格以 Markdown 呈現)
|
||||
- md-i(表格以圖片呈現)
|
||||
|
||||
輸出格式:tar.gz
|
||||
輸出格式:.tar(不壓縮封裝)
|
||||
|
||||
---
|
||||
|
||||
## 檔案傳輸機制
|
||||
|
||||
Contents.CN 使用統一的檔案傳輸策略:
|
||||
|
||||
| 檔案大小 | 傳輸方式 | 說明 |
|
||||
| -------- | -------- | ---- |
|
||||
| ≤ 10MB | 直接傳輸 | 單一請求完成上傳/下載 |
|
||||
| > 10MB | 分段傳輸 | 使用 5MB chunks 分段傳輸 |
|
||||
|
||||
### 多檔輸出封裝
|
||||
|
||||
- ✅ 唯一允許格式:`.tar`
|
||||
- ❌ 禁止使用:`.tar.gz`、`.tgz`、`.zip`
|
||||
|
||||
### 設計原則
|
||||
|
||||
- 分段傳輸僅存在於傳輸層
|
||||
- 轉換引擎只接收完整檔案
|
||||
- 前後端使用共用傳輸模組
|
||||
|
||||
---
|
||||
|
||||
|
|
|
|||
134
public/script.js
134
public/script.js
|
|
@ -242,11 +242,54 @@ const deleteRow = (target) => {
|
|||
}).catch((err) => console.log(err));
|
||||
};
|
||||
|
||||
// ==================== 全域傳輸常數 ====================
|
||||
const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
|
||||
const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
/**
|
||||
* 判斷是否需要使用 chunk 傳輸
|
||||
*/
|
||||
function shouldUseChunkedUpload(fileSize) {
|
||||
return fileSize > CHUNK_THRESHOLD_BYTES;
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成 UUID
|
||||
*/
|
||||
function generateUploadId() {
|
||||
if (typeof crypto !== "undefined" && crypto.randomUUID) {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) {
|
||||
const r = Math.random() * 16 | 0;
|
||||
const v = c === "x" ? r : (r & 0x3 | 0x8);
|
||||
return v.toString(16);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 統一上傳檔案(自動判斷使用直傳或 chunk)
|
||||
*
|
||||
* @param {File} file - 要上傳的檔案
|
||||
*/
|
||||
const uploadFile = (file) => {
|
||||
convertButton.disabled = true;
|
||||
convertButton.value = getTranslation("convert", "uploading");
|
||||
pendingFiles += 1;
|
||||
|
||||
if (shouldUseChunkedUpload(file.size)) {
|
||||
// 大檔:使用 chunk 上傳
|
||||
uploadFileChunked(file);
|
||||
} else {
|
||||
// 小檔:直接上傳
|
||||
uploadFileDirect(file);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 直接上傳(≤10MB)
|
||||
*/
|
||||
const uploadFileDirect = (file) => {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file, file.name);
|
||||
|
||||
|
|
@ -255,7 +298,12 @@ const uploadFile = (file) => {
|
|||
xhr.open("POST", `${webroot}/upload`, true);
|
||||
|
||||
xhr.onload = () => {
|
||||
let data = JSON.parse(xhr.responseText);
|
||||
let data = {};
|
||||
try {
|
||||
data = JSON.parse(xhr.responseText);
|
||||
} catch (e) {
|
||||
console.log("Parse error:", e);
|
||||
}
|
||||
|
||||
pendingFiles -= 1;
|
||||
if (pendingFiles === 0) {
|
||||
|
|
@ -265,10 +313,12 @@ const uploadFile = (file) => {
|
|||
convertButton.value = getTranslation("convert", "convertButton");
|
||||
}
|
||||
|
||||
//Remove the progress bar when upload is done
|
||||
// Remove the progress bar when upload is done
|
||||
let progressbar = file.htmlRow.getElementsByTagName("progress");
|
||||
progressbar[0].parentElement.remove();
|
||||
console.log(data);
|
||||
if (progressbar[0]) {
|
||||
progressbar[0].parentElement.remove();
|
||||
}
|
||||
console.log("Direct upload complete:", data);
|
||||
};
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
|
|
@ -277,16 +327,88 @@ const uploadFile = (file) => {
|
|||
console.log(`upload progress (${file.name}):`, (100 * sent) / total);
|
||||
|
||||
let progressbar = file.htmlRow.getElementsByTagName("progress");
|
||||
progressbar[0].value = (100 * sent) / total;
|
||||
if (progressbar[0]) {
|
||||
progressbar[0].value = (100 * sent) / total;
|
||||
}
|
||||
};
|
||||
|
||||
xhr.onerror = (e) => {
|
||||
console.log(e);
|
||||
console.log("Upload error:", e);
|
||||
pendingFiles -= 1;
|
||||
if (pendingFiles === 0) {
|
||||
convertButton.value = getTranslation("convert", "convertButton");
|
||||
}
|
||||
};
|
||||
|
||||
xhr.send(formData);
|
||||
};
|
||||
|
||||
/**
|
||||
* Chunk 上傳(>10MB)
|
||||
*/
|
||||
const uploadFileChunked = async (file) => {
|
||||
const uploadId = generateUploadId();
|
||||
const totalChunks = Math.ceil(file.size / CHUNK_SIZE_BYTES);
|
||||
|
||||
console.log(`Starting chunked upload: ${file.name}, size: ${file.size}, chunks: ${totalChunks}`);
|
||||
|
||||
try {
|
||||
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
||||
const start = chunkIndex * CHUNK_SIZE_BYTES;
|
||||
const end = Math.min(start + CHUNK_SIZE_BYTES, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("upload_id", uploadId);
|
||||
formData.append("chunk_index", chunkIndex.toString());
|
||||
formData.append("total_chunks", totalChunks.toString());
|
||||
formData.append("file_name", file.name);
|
||||
formData.append("total_size", file.size.toString());
|
||||
formData.append("chunk", chunk);
|
||||
|
||||
const response = await fetch(`${webroot}/upload-chunk`, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`);
|
||||
}
|
||||
|
||||
// 更新進度
|
||||
const percent = ((chunkIndex + 1) / totalChunks) * 100;
|
||||
let progressbar = file.htmlRow.getElementsByTagName("progress");
|
||||
if (progressbar[0]) {
|
||||
progressbar[0].value = percent;
|
||||
}
|
||||
console.log(`Chunk ${chunkIndex + 1}/${totalChunks} uploaded (${percent.toFixed(1)}%)`);
|
||||
}
|
||||
|
||||
// 完成
|
||||
pendingFiles -= 1;
|
||||
if (pendingFiles === 0) {
|
||||
if (formatSelected) {
|
||||
convertButton.disabled = false;
|
||||
}
|
||||
convertButton.value = getTranslation("convert", "convertButton");
|
||||
}
|
||||
|
||||
// Remove the progress bar
|
||||
let progressbar = file.htmlRow.getElementsByTagName("progress");
|
||||
if (progressbar[0]) {
|
||||
progressbar[0].parentElement.remove();
|
||||
}
|
||||
console.log("Chunked upload complete:", file.name);
|
||||
|
||||
} catch (error) {
|
||||
console.error("Chunked upload failed:", error);
|
||||
pendingFiles -= 1;
|
||||
if (pendingFiles === 0) {
|
||||
convertButton.value = getTranslation("convert", "convertButton");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const formConvert = document.querySelector(`form[action='${webroot}/convert']`);
|
||||
|
||||
formConvert.addEventListener("submit", () => {
|
||||
|
|
|
|||
388
public/transfer.js
Normal file
388
public/transfer.js
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
/**
|
||||
* Contents.CN 前端檔案傳輸管理器
|
||||
*
|
||||
* 統一處理所有檔案的上傳與下載:
|
||||
* - 檔案 ≤ 10MB:直接傳輸
|
||||
* - 檔案 > 10MB:使用 chunk 分段傳輸
|
||||
*
|
||||
* ⚠️ 重要:所有功能必須使用此模組,不得自行實作傳輸邏輯
|
||||
*/
|
||||
|
||||
// ==================== 常數定義 ====================
|
||||
|
||||
/**
|
||||
* 檔案大小門檻(10MB)
|
||||
*/
|
||||
const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* 每個 chunk 的大小(5MB)
|
||||
*/
|
||||
const CHUNK_SIZE_BYTES = 5 * 1024 * 1024;
|
||||
|
||||
// ==================== 工具函數 ====================
|
||||
|
||||
/**
|
||||
* 生成 UUID(用於 upload_id)
|
||||
*/
|
||||
function generateUploadId() {
|
||||
if (typeof crypto !== "undefined" && crypto.randomUUID) {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
// 降級方案
|
||||
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) {
|
||||
const r = Math.random() * 16 | 0;
|
||||
const v = c === "x" ? r : (r & 0x3 | 0x8);
|
||||
return v.toString(16);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 判斷是否需要使用 chunk 傳輸
|
||||
*/
|
||||
function shouldUseChunkedTransfer(fileSize) {
|
||||
return fileSize > CHUNK_THRESHOLD_BYTES;
|
||||
}
|
||||
|
||||
/**
|
||||
* 計算 chunk 數量
|
||||
*/
|
||||
function calculateChunkCount(fileSize) {
|
||||
return Math.ceil(fileSize / CHUNK_SIZE_BYTES);
|
||||
}
|
||||
|
||||
// ==================== 上傳管理器 ====================
|
||||
|
||||
/**
|
||||
* 上傳管理器類別
|
||||
*/
|
||||
class UploadManager {
|
||||
/**
|
||||
* @param {string} webroot - 網站根路徑
|
||||
*/
|
||||
constructor(webroot) {
|
||||
this.webroot = webroot;
|
||||
this.activeUploads = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
* 上傳檔案(自動判斷使用直傳或 chunk)
|
||||
*
|
||||
* @param {File} file - 要上傳的檔案
|
||||
* @param {object} options - 選項
|
||||
* @param {function} options.onProgress - 進度回調 (percent: number) => void
|
||||
* @param {function} options.onComplete - 完成回調 (response: object) => void
|
||||
* @param {function} options.onError - 錯誤回調 (error: Error) => void
|
||||
* @returns {Promise<object>} 上傳結果
|
||||
*/
|
||||
async uploadFile(file, options = {}) {
|
||||
const { onProgress, onComplete, onError } = options;
|
||||
|
||||
try {
|
||||
let result;
|
||||
|
||||
if (shouldUseChunkedTransfer(file.size)) {
|
||||
// 大檔:使用 chunk 上傳
|
||||
result = await this.uploadChunked(file, onProgress);
|
||||
} else {
|
||||
// 小檔:直接上傳
|
||||
result = await this.uploadDirect(file, onProgress);
|
||||
}
|
||||
|
||||
if (onComplete) onComplete(result);
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (onError) onError(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接上傳(小檔)
|
||||
*/
|
||||
async uploadDirect(file, onProgress) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file, file.name);
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", `${this.webroot}/upload`, true);
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
if (e.lengthComputable && onProgress) {
|
||||
const percent = (e.loaded / e.total) * 100;
|
||||
onProgress(percent);
|
||||
}
|
||||
};
|
||||
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
try {
|
||||
const data = JSON.parse(xhr.responseText);
|
||||
resolve(data);
|
||||
} catch {
|
||||
resolve({ success: true, message: "Upload completed" });
|
||||
}
|
||||
} else {
|
||||
reject(new Error(`Upload failed with status ${xhr.status}`));
|
||||
}
|
||||
};
|
||||
|
||||
xhr.onerror = () => reject(new Error("Upload failed"));
|
||||
xhr.send(formData);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 上傳(大檔)
|
||||
*/
|
||||
async uploadChunked(file, onProgress) {
|
||||
const uploadId = generateUploadId();
|
||||
const totalChunks = calculateChunkCount(file.size);
|
||||
|
||||
this.activeUploads.set(uploadId, {
|
||||
file,
|
||||
totalChunks,
|
||||
uploadedChunks: 0,
|
||||
status: "uploading"
|
||||
});
|
||||
|
||||
try {
|
||||
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
||||
const start = chunkIndex * CHUNK_SIZE_BYTES;
|
||||
const end = Math.min(start + CHUNK_SIZE_BYTES, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
|
||||
await this.uploadChunk(uploadId, chunkIndex, totalChunks, chunk, file.name, file.size);
|
||||
|
||||
// 更新進度
|
||||
const uploadInfo = this.activeUploads.get(uploadId);
|
||||
if (uploadInfo) {
|
||||
uploadInfo.uploadedChunks = chunkIndex + 1;
|
||||
const percent = ((chunkIndex + 1) / totalChunks) * 100;
|
||||
if (onProgress) onProgress(percent);
|
||||
}
|
||||
}
|
||||
|
||||
this.activeUploads.delete(uploadId);
|
||||
return { success: true, message: "Chunked upload completed", upload_id: uploadId };
|
||||
} catch (error) {
|
||||
this.activeUploads.delete(uploadId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 上傳單個 chunk
|
||||
*/
|
||||
async uploadChunk(uploadId, chunkIndex, totalChunks, chunkData, fileName, totalSize) {
|
||||
const formData = new FormData();
|
||||
formData.append("upload_id", uploadId);
|
||||
formData.append("chunk_index", chunkIndex.toString());
|
||||
formData.append("total_chunks", totalChunks.toString());
|
||||
formData.append("file_name", fileName);
|
||||
formData.append("total_size", totalSize.toString());
|
||||
formData.append("chunk", chunkData);
|
||||
|
||||
const response = await fetch(`${this.webroot}/upload-chunk`, {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* 取消上傳
|
||||
*/
|
||||
cancelUpload(uploadId) {
|
||||
const upload = this.activeUploads.get(uploadId);
|
||||
if (upload) {
|
||||
upload.status = "cancelled";
|
||||
this.activeUploads.delete(uploadId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 下載管理器 ====================
|
||||
|
||||
/**
|
||||
* 下載管理器類別
|
||||
*/
|
||||
class DownloadManager {
|
||||
/**
|
||||
* @param {string} webroot - 網站根路徑
|
||||
*/
|
||||
constructor(webroot) {
|
||||
this.webroot = webroot;
|
||||
}
|
||||
|
||||
/**
|
||||
* 下載檔案(自動判斷使用直傳或 chunk)
|
||||
*
|
||||
* @param {string} url - 下載 URL
|
||||
* @param {string} fileName - 檔案名稱
|
||||
* @param {object} options - 選項
|
||||
* @param {function} options.onProgress - 進度回調
|
||||
* @returns {Promise<Blob>} 下載的檔案
|
||||
*/
|
||||
async downloadFile(url, fileName, options = {}) {
|
||||
const { onProgress } = options;
|
||||
|
||||
// 先取得檔案資訊
|
||||
const info = await this.getFileInfo(url);
|
||||
|
||||
if (!info) {
|
||||
// 無法取得資訊,使用直接下載
|
||||
return this.downloadDirect(url, fileName);
|
||||
}
|
||||
|
||||
if (shouldUseChunkedTransfer(info.total_size)) {
|
||||
// 大檔:使用 chunk 下載
|
||||
return this.downloadChunked(url, fileName, info, onProgress);
|
||||
} else {
|
||||
// 小檔:直接下載
|
||||
return this.downloadDirect(url, fileName, onProgress);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得檔案資訊
|
||||
*/
|
||||
async getFileInfo(url) {
|
||||
try {
|
||||
const response = await fetch(`${url}/info`, { method: "GET" });
|
||||
if (response.ok) {
|
||||
return response.json();
|
||||
}
|
||||
} catch {
|
||||
// 忽略錯誤,降級為直接下載
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接下載(小檔)
|
||||
*/
|
||||
async downloadDirect(url, fileName, onProgress) {
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Download failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get("Content-Length");
|
||||
const total = contentLength ? parseInt(contentLength, 10) : 0;
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const chunks = [];
|
||||
let loaded = 0;
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
chunks.push(value);
|
||||
loaded += value.length;
|
||||
|
||||
if (onProgress && total > 0) {
|
||||
onProgress((loaded / total) * 100);
|
||||
}
|
||||
}
|
||||
|
||||
const blob = new Blob(chunks);
|
||||
this.triggerDownload(blob, fileName);
|
||||
return blob;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 下載(大檔)
|
||||
*/
|
||||
async downloadChunked(url, fileName, info, onProgress) {
|
||||
const { total_chunks, chunk_size, total_size } = info;
|
||||
const chunks = [];
|
||||
|
||||
for (let i = 0; i < total_chunks; i++) {
|
||||
const chunkData = await this.downloadChunk(url, i);
|
||||
chunks.push(chunkData);
|
||||
|
||||
if (onProgress) {
|
||||
const loaded = Math.min((i + 1) * chunk_size, total_size);
|
||||
onProgress((loaded / total_size) * 100);
|
||||
}
|
||||
}
|
||||
|
||||
const blob = new Blob(chunks);
|
||||
this.triggerDownload(blob, fileName);
|
||||
return blob;
|
||||
}
|
||||
|
||||
/**
|
||||
* 下載單個 chunk
|
||||
*/
|
||||
async downloadChunk(url, chunkIndex) {
|
||||
const response = await fetch(`${url}/chunk/${chunkIndex}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Chunk ${chunkIndex} download failed: ${response.status}`);
|
||||
}
|
||||
|
||||
return response.arrayBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* 觸發瀏覽器下載
|
||||
*/
|
||||
triggerDownload(blob, fileName) {
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = fileName;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* 簡單下載(不使用 chunk,用於向後相容)
|
||||
*/
|
||||
async simpleDownload(url, fileName) {
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = fileName;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 全域實例 ====================
|
||||
|
||||
// 從 meta 標籤取得 webroot
|
||||
const webrootMeta = document.querySelector("meta[name='webroot']");
|
||||
const webroot = webrootMeta ? webrootMeta.content : "";
|
||||
|
||||
// 建立全域實例
|
||||
window.ContentsTransfer = {
|
||||
uploadManager: new UploadManager(webroot),
|
||||
downloadManager: new DownloadManager(webroot),
|
||||
|
||||
// 工具函數
|
||||
shouldUseChunkedTransfer,
|
||||
calculateChunkCount,
|
||||
|
||||
// 常數
|
||||
CHUNK_THRESHOLD_BYTES,
|
||||
CHUNK_SIZE_BYTES,
|
||||
};
|
||||
|
||||
// 向後相容:提供簡化的 API
|
||||
window.uploadFile = (file, options) => window.ContentsTransfer.uploadManager.uploadFile(file, options);
|
||||
window.downloadFile = (url, fileName, options) => window.ContentsTransfer.downloadManager.downloadFile(url, fileName, options);
|
||||
|
||||
console.log("Contents.CN Transfer Module initialized");
|
||||
|
|
@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process";
|
|||
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs";
|
||||
import { join, basename, dirname } from "node:path";
|
||||
import { ExecFileFn } from "./types";
|
||||
import { createConverterArchive, getArchiveFileName } from "../transfer";
|
||||
|
||||
export const properties = {
|
||||
from: {
|
||||
|
|
@ -14,19 +15,22 @@ export const properties = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Helper function to create a tar.gz archive from a directory
|
||||
* Helper function to create a .tar archive from a directory (no compression)
|
||||
*
|
||||
* ⚠️ 重要:僅使用 .tar 格式,禁止 .tar.gz / .tgz / .zip
|
||||
*/
|
||||
function createTarGzArchive(
|
||||
function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputTarGz: string,
|
||||
outputTar: string,
|
||||
execFile: ExecFileFn,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use tar command to create gzipped archive
|
||||
// tar -czf <output.tar.gz> -C <sourceDir> .
|
||||
// Use tar command to create archive (without gzip compression)
|
||||
// tar -cf <output.tar> -C <sourceDir> .
|
||||
// 注意:使用 -cf 而非 -czf,避免 gzip 壓縮
|
||||
execFile(
|
||||
"tar",
|
||||
["-czf", outputTarGz, "-C", sourceDir, "."],
|
||||
["-cf", outputTar, "-C", sourceDir, "."],
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`tar error: ${error}`);
|
||||
|
|
@ -122,15 +126,14 @@ export async function convert(
|
|||
// MinerU outputs to a subdirectory, find the actual output
|
||||
const mineruActualOutput = join(mineruOutputDir, "auto");
|
||||
|
||||
// Create tar.gz archive from the output directory
|
||||
const tarGzPath = targetPath.endsWith(".tar.gz")
|
||||
? targetPath
|
||||
: `${targetPath}.tar.gz`;
|
||||
// Create .tar archive from the output directory (不使用壓縮)
|
||||
// 強制使用 .tar 格式,禁止 .tar.gz
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
|
||||
// Ensure the parent directory exists
|
||||
const tarGzDir = dirname(tarGzPath);
|
||||
if (!existsSync(tarGzDir)) {
|
||||
mkdirSync(tarGzDir, { recursive: true });
|
||||
const tarDir = dirname(tarPath);
|
||||
if (!existsSync(tarDir)) {
|
||||
mkdirSync(tarDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Use the actual MinerU output directory for archiving
|
||||
|
|
@ -138,14 +141,14 @@ export async function convert(
|
|||
? mineruActualOutput
|
||||
: mineruOutputDir;
|
||||
|
||||
await createTarGzArchive(outputToArchive, tarGzPath, execFile);
|
||||
await createTarArchive(outputToArchive, tarPath, execFile);
|
||||
|
||||
// Clean up the temporary directory
|
||||
removeDir(mineruOutputDir);
|
||||
|
||||
resolve("Done");
|
||||
} catch (tarError) {
|
||||
reject(`Failed to create tar.gz archive: ${tarError}`);
|
||||
reject(`Failed to create .tar archive: ${tarError}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ import { convert } from "./pages/convert";
|
|||
import { deleteFile } from "./pages/deleteFile";
|
||||
import { deleteJob } from "./pages/deleteJob";
|
||||
import { download } from "./pages/download";
|
||||
import { downloadChunk } from "./pages/downloadChunk";
|
||||
import { history } from "./pages/history";
|
||||
import { listConverters } from "./pages/listConverters";
|
||||
import { results } from "./pages/results";
|
||||
import { root } from "./pages/root";
|
||||
import { upload } from "./pages/upload";
|
||||
import { uploadChunk, uploadInfo } from "./pages/uploadChunk";
|
||||
import { user } from "./pages/user";
|
||||
import { healthcheck } from "./pages/healthcheck";
|
||||
|
||||
|
|
@ -41,9 +43,12 @@ const app = new Elysia({
|
|||
.use(user)
|
||||
.use(root)
|
||||
.use(upload)
|
||||
.use(uploadChunk)
|
||||
.use(uploadInfo)
|
||||
.use(history)
|
||||
.use(convert)
|
||||
.use(download)
|
||||
.use(downloadChunk)
|
||||
.use(deleteJob)
|
||||
.use(results)
|
||||
.use(deleteFile)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import path from "node:path";
|
||||
import { Elysia } from "elysia";
|
||||
import sanitize from "sanitize-filename";
|
||||
import * as tar from "tar";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import { createJobArchive, shouldUseChunkedDownload } from "../transfer";
|
||||
|
||||
export const download = new Elysia()
|
||||
.use(userService)
|
||||
|
|
@ -45,18 +45,10 @@ export const download = new Elysia()
|
|||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const outputPath = `${outputDir}${userId}/${jobId}`;
|
||||
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
|
||||
|
||||
await tar.create(
|
||||
{
|
||||
file: outputTar,
|
||||
cwd: outputPath,
|
||||
filter: (path) => {
|
||||
return !path.match(".*\\.tar");
|
||||
},
|
||||
},
|
||||
["."],
|
||||
);
|
||||
// 使用統一的封裝管理器建立 .tar(不壓縮)
|
||||
const outputTar = await createJobArchive(outputPath, jobId);
|
||||
|
||||
return Bun.file(outputTar);
|
||||
},
|
||||
{
|
||||
|
|
|
|||
178
src/pages/downloadChunk.tsx
Normal file
178
src/pages/downloadChunk.tsx
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Contents.CN Chunk 下載 API
|
||||
*
|
||||
* 處理大檔的分段下載
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { join } from "node:path";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
createChunkDownloadHeaders,
|
||||
getFileForDirectDownload,
|
||||
createDirectDownloadHeaders,
|
||||
CHUNK_SIZE_BYTES,
|
||||
} from "../transfer";
|
||||
import { statSync, existsSync } from "node:fs";
|
||||
|
||||
export const downloadChunk = new Elysia()
|
||||
.use(userService)
|
||||
/**
|
||||
* 取得檔案下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(filePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* 下載特定 chunk
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(filePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
if (!existsSync(archivePath)) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(archivePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(archivePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
);
|
||||
88
src/pages/uploadChunk.tsx
Normal file
88
src/pages/uploadChunk.tsx
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/**
|
||||
* Contents.CN Chunk 上傳 API
|
||||
*
|
||||
* 處理大檔的分段上傳
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { uploadsDir } from "../index";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
handleChunkUpload,
|
||||
shouldUseChunkedUpload,
|
||||
CHUNK_SIZE_BYTES,
|
||||
calculateChunkCount,
|
||||
} from "../transfer";
|
||||
|
||||
export const uploadChunk = new Elysia().use(userService).post(
|
||||
"/upload-chunk",
|
||||
async ({ body, user, cookie: { jobId } }) => {
|
||||
if (!jobId?.value) {
|
||||
return {
|
||||
success: false,
|
||||
message: "No active job session",
|
||||
};
|
||||
}
|
||||
|
||||
const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body;
|
||||
|
||||
const sanitizedFileName = sanitize(file_name);
|
||||
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
|
||||
|
||||
// 取得 chunk 資料
|
||||
const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk;
|
||||
|
||||
const result = await handleChunkUpload(
|
||||
upload_id,
|
||||
parseInt(chunk_index, 10),
|
||||
parseInt(total_chunks, 10),
|
||||
chunkData,
|
||||
sanitizedFileName,
|
||||
parseInt(total_size, 10),
|
||||
user.id,
|
||||
jobId.value,
|
||||
`${uploadsDir}${user.id}/`,
|
||||
userUploadsDir
|
||||
);
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
upload_id: t.String(),
|
||||
chunk_index: t.String(),
|
||||
total_chunks: t.String(),
|
||||
file_name: t.String(),
|
||||
total_size: t.String(),
|
||||
chunk: t.File(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* 取得上傳資訊(用於前端判斷是否需要 chunk 上傳)
|
||||
*/
|
||||
export const uploadInfo = new Elysia().use(userService).post(
|
||||
"/upload-info",
|
||||
async ({ body }) => {
|
||||
const { file_size } = body;
|
||||
const size = parseInt(file_size, 10);
|
||||
|
||||
const useChunked = shouldUseChunkedUpload(size);
|
||||
|
||||
return {
|
||||
use_chunked: useChunked,
|
||||
chunk_size: CHUNK_SIZE_BYTES,
|
||||
total_chunks: useChunked ? calculateChunkCount(size) : 1,
|
||||
};
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
file_size: t.String(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
154
src/transfer/archiveManager.ts
Normal file
154
src/transfer/archiveManager.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
/**
|
||||
* Contents.CN 封裝管理器
|
||||
*
|
||||
* 統一處理多檔輸出的封裝:
|
||||
* - 唯一允許格式:.tar
|
||||
* - 禁止:.tar.gz, .tgz, .zip 等
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import * as tar from "tar";
|
||||
import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants";
|
||||
|
||||
/**
|
||||
* 驗證封裝格式是否合法
|
||||
*/
|
||||
export function validateArchiveFormat(fileName: string): boolean {
|
||||
const lowerName = fileName.toLowerCase();
|
||||
|
||||
// 檢查是否使用禁止的格式
|
||||
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
|
||||
if (lowerName.endsWith(forbidden)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 檢查是否是允許的格式
|
||||
return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得正確的封裝檔名(強制使用 .tar)
|
||||
*/
|
||||
export function getArchiveFileName(baseName: string): string {
|
||||
const lowerName = baseName.toLowerCase();
|
||||
|
||||
// 移除任何禁止的副檔名
|
||||
let cleanName = baseName;
|
||||
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
|
||||
if (lowerName.endsWith(forbidden)) {
|
||||
cleanName = baseName.slice(0, -forbidden.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果已經是 .tar 結尾,直接返回
|
||||
if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) {
|
||||
return cleanName;
|
||||
}
|
||||
|
||||
// 否則加上 .tar
|
||||
return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立 .tar 封裝(不壓縮)
|
||||
*
|
||||
* @param sourceDir - 來源目錄
|
||||
* @param outputPath - 輸出的 .tar 檔案路徑
|
||||
* @param options - 額外選項
|
||||
*/
|
||||
export async function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputPath: string,
|
||||
options: {
|
||||
filter?: (path: string) => boolean;
|
||||
prefix?: string;
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
// 強制確保輸出是 .tar 格式
|
||||
const finalOutputPath = getArchiveFileName(outputPath);
|
||||
|
||||
// 確保輸出目錄存在
|
||||
const outputDir = join(finalOutputPath, "..");
|
||||
if (!existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// 取得要封裝的檔案列表
|
||||
const files = readdirSync(sourceDir);
|
||||
|
||||
// 預設過濾器:排除其他 tar 檔案
|
||||
const defaultFilter = (path: string) => !path.match(/\.tar$/i);
|
||||
const filter = options.filter || defaultFilter;
|
||||
|
||||
await tar.create(
|
||||
{
|
||||
file: finalOutputPath,
|
||||
cwd: sourceDir,
|
||||
filter: filter,
|
||||
// 不使用 gzip 壓縮
|
||||
gzip: false,
|
||||
},
|
||||
files.filter(f => filter(f))
|
||||
);
|
||||
|
||||
console.log(`Created tar archive: ${finalOutputPath}`);
|
||||
return finalOutputPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立用於多檔輸出的 .tar 封裝
|
||||
*
|
||||
* @param outputDir - 輸出目錄(包含多個轉換結果)
|
||||
* @param jobId - 任務 ID
|
||||
*/
|
||||
export async function createJobArchive(outputDir: string, jobId: string): Promise<string> {
|
||||
const archiveName = `converted_files_${jobId}.tar`;
|
||||
const archivePath = join(outputDir, archiveName);
|
||||
|
||||
await createTarArchive(outputDir, archivePath, {
|
||||
filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案
|
||||
});
|
||||
|
||||
return archivePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 從目錄封裝 MinerU 等多檔輸出引擎的結果
|
||||
*
|
||||
* @param mineruOutputDir - MinerU 輸出目錄
|
||||
* @param targetPath - 目標 .tar 路徑
|
||||
*/
|
||||
export async function createConverterArchive(
|
||||
converterOutputDir: string,
|
||||
targetPath: string
|
||||
): Promise<string> {
|
||||
// 確保輸出使用 .tar 格式
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
|
||||
await createTarArchive(converterOutputDir, tarPath);
|
||||
|
||||
return tarPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得封裝檔案資訊
|
||||
*/
|
||||
export function getArchiveInfo(archivePath: string): {
|
||||
exists: boolean;
|
||||
size: number;
|
||||
fileName: string;
|
||||
} | null {
|
||||
if (!existsSync(archivePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(archivePath);
|
||||
return {
|
||||
exists: true,
|
||||
size: stats.size,
|
||||
fileName: basename(archivePath),
|
||||
};
|
||||
}
|
||||
38
src/transfer/constants.ts
Normal file
38
src/transfer/constants.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸常數
|
||||
*
|
||||
* 這些常數定義了整個系統的檔案傳輸規則
|
||||
*/
|
||||
|
||||
/**
|
||||
* 檔案大小門檻(10MB)
|
||||
* - 小於等於此值:直接傳輸
|
||||
* - 大於此值:使用 chunk 分段傳輸
|
||||
*/
|
||||
export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
|
||||
|
||||
/**
|
||||
* 每個 chunk 的大小(5MB)
|
||||
*/
|
||||
export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
/**
|
||||
* 上傳 session 過期時間(毫秒)
|
||||
* 預設 1 小時
|
||||
*/
|
||||
export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
/**
|
||||
* 暫存目錄名稱
|
||||
*/
|
||||
export const CHUNK_TEMP_DIR = "chunks_temp";
|
||||
|
||||
/**
|
||||
* 允許的封裝格式(僅 .tar)
|
||||
*/
|
||||
export const ALLOWED_ARCHIVE_FORMAT = ".tar";
|
||||
|
||||
/**
|
||||
* 禁止的封裝格式
|
||||
*/
|
||||
export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"];
|
||||
123
src/transfer/downloadManager.ts
Normal file
123
src/transfer/downloadManager.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Contents.CN 後端下載管理器
|
||||
*
|
||||
* 統一處理所有檔案下載,包含:
|
||||
* - 小檔(≤10MB):直接回傳
|
||||
* - 大檔(>10MB):chunk 分段下載
|
||||
*/
|
||||
|
||||
import { existsSync, statSync, createReadStream } from "node:fs";
|
||||
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants";
|
||||
import type { ChunkDownloadInfo } from "./types";
|
||||
import { getTransferMode } from "./types";
|
||||
import { basename } from "node:path";
|
||||
|
||||
/**
|
||||
* 判斷檔案是否需要使用 chunk 下載
|
||||
*/
|
||||
export function shouldUseChunkedDownload(filePath: string): boolean {
|
||||
if (!existsSync(filePath)) {
|
||||
return false;
|
||||
}
|
||||
const stats = statSync(filePath);
|
||||
return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked";
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得檔案的 chunk 下載資訊
|
||||
*/
|
||||
export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(filePath);
|
||||
const fileName = basename(filePath);
|
||||
|
||||
return {
|
||||
total_size: stats.size,
|
||||
total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES),
|
||||
chunk_size: CHUNK_SIZE_BYTES,
|
||||
file_name: fileName,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得特定 chunk 的資料
|
||||
*/
|
||||
export async function getChunk(
|
||||
filePath: string,
|
||||
chunkIndex: number,
|
||||
chunkSize: number = CHUNK_SIZE_BYTES
|
||||
): Promise<Buffer | null> {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(filePath);
|
||||
const start = chunkIndex * chunkSize;
|
||||
const end = Math.min(start + chunkSize, stats.size);
|
||||
|
||||
if (start >= stats.size) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const stream = createReadStream(filePath, { start, end: end - 1 });
|
||||
|
||||
stream.on("data", (chunk) => {
|
||||
chunks.push(chunk as Buffer);
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
resolve(Buffer.concat(chunks));
|
||||
});
|
||||
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接取得完整檔案(小檔用)
|
||||
*/
|
||||
export function getFileForDirectDownload(filePath: string): ReturnType<typeof Bun.file> | null {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
return Bun.file(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立 chunk 下載回應的 headers
|
||||
*/
|
||||
export function createChunkDownloadHeaders(
|
||||
info: ChunkDownloadInfo,
|
||||
chunkIndex: number,
|
||||
chunkData: Buffer
|
||||
): Record<string, string> {
|
||||
const start = chunkIndex * info.chunk_size;
|
||||
const end = start + chunkData.length - 1;
|
||||
|
||||
return {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": chunkData.length.toString(),
|
||||
"Content-Range": `bytes ${start}-${end}/${info.total_size}`,
|
||||
"Accept-Ranges": "bytes",
|
||||
"X-Chunk-Index": chunkIndex.toString(),
|
||||
"X-Total-Chunks": info.total_chunks.toString(),
|
||||
"X-File-Name": encodeURIComponent(info.file_name),
|
||||
"X-Total-Size": info.total_size.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立直接下載回應的 headers
|
||||
*/
|
||||
export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record<string, string> {
|
||||
return {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": fileSize.toString(),
|
||||
"Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`,
|
||||
};
|
||||
}
|
||||
56
src/transfer/index.ts
Normal file
56
src/transfer/index.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸模組
|
||||
*
|
||||
* 統一匯出所有傳輸相關功能
|
||||
*/
|
||||
|
||||
// 常數
|
||||
export {
|
||||
CHUNK_THRESHOLD_BYTES,
|
||||
CHUNK_SIZE_BYTES,
|
||||
UPLOAD_SESSION_TIMEOUT_MS,
|
||||
CHUNK_TEMP_DIR,
|
||||
ALLOWED_ARCHIVE_FORMAT,
|
||||
FORBIDDEN_ARCHIVE_FORMATS,
|
||||
} from "./constants";
|
||||
|
||||
// 類型
|
||||
export type {
|
||||
ChunkUploadRequest,
|
||||
ChunkUploadResponse,
|
||||
DirectUploadResponse,
|
||||
ChunkDownloadInfo,
|
||||
UploadSession,
|
||||
TransferMode,
|
||||
} from "./types";
|
||||
|
||||
export { getTransferMode } from "./types";
|
||||
|
||||
// 上傳管理
|
||||
export {
|
||||
uploadSessionManager,
|
||||
shouldUseChunkedUpload,
|
||||
handleDirectUpload,
|
||||
handleChunkUpload,
|
||||
calculateChunkCount,
|
||||
} from "./uploadManager";
|
||||
|
||||
// 下載管理
|
||||
export {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
getFileForDirectDownload,
|
||||
createChunkDownloadHeaders,
|
||||
createDirectDownloadHeaders,
|
||||
} from "./downloadManager";
|
||||
|
||||
// 封裝管理
|
||||
export {
|
||||
validateArchiveFormat,
|
||||
getArchiveFileName,
|
||||
createTarArchive,
|
||||
createJobArchive,
|
||||
createConverterArchive,
|
||||
getArchiveInfo,
|
||||
} from "./archiveManager";
|
||||
94
src/transfer/types.ts
Normal file
94
src/transfer/types.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* Contents.CN 檔案傳輸類型定義
|
||||
*/
|
||||
|
||||
/**
|
||||
* Chunk 上傳請求結構
|
||||
*/
|
||||
export interface ChunkUploadRequest {
|
||||
/** 上傳會話 ID(UUID) */
|
||||
upload_id: string;
|
||||
/** 當前 chunk 索引(從 0 開始) */
|
||||
chunk_index: number;
|
||||
/** 總 chunk 數量 */
|
||||
total_chunks: number;
|
||||
/** chunk 數據 */
|
||||
data: Blob | ArrayBuffer;
|
||||
/** 原始檔案名稱 */
|
||||
file_name: string;
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 上傳回應結構
|
||||
*/
|
||||
export interface ChunkUploadResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
/** 已接收的 chunks 索引列表 */
|
||||
received_chunks?: number[];
|
||||
/** 是否所有 chunks 都已接收完成 */
|
||||
completed?: boolean;
|
||||
/** 合併後的檔案路徑(僅完成時返回) */
|
||||
file_path?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接上傳回應結構
|
||||
*/
|
||||
export interface DirectUploadResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
file_path?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 下載資訊
|
||||
*/
|
||||
export interface ChunkDownloadInfo {
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
/** chunk 數量 */
|
||||
total_chunks: number;
|
||||
/** 每個 chunk 大小 */
|
||||
chunk_size: number;
|
||||
/** 檔案名稱 */
|
||||
file_name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 上傳會話狀態
|
||||
*/
|
||||
export interface UploadSession {
|
||||
/** 上傳 ID */
|
||||
upload_id: string;
|
||||
/** 使用者 ID */
|
||||
user_id: string;
|
||||
/** Job ID */
|
||||
job_id: string;
|
||||
/** 檔案名稱 */
|
||||
file_name: string;
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
/** 總 chunk 數 */
|
||||
total_chunks: number;
|
||||
/** 已接收的 chunks */
|
||||
received_chunks: Set<number>;
|
||||
/** 建立時間 */
|
||||
created_at: Date;
|
||||
/** 暫存目錄路徑 */
|
||||
temp_dir: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 傳輸模式
|
||||
*/
|
||||
export type TransferMode = "direct" | "chunked";
|
||||
|
||||
/**
|
||||
* 根據檔案大小決定傳輸模式
|
||||
*/
|
||||
export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode {
|
||||
return fileSize <= thresholdBytes ? "direct" : "chunked";
|
||||
}
|
||||
303
src/transfer/uploadManager.ts
Normal file
303
src/transfer/uploadManager.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
/**
|
||||
* Contents.CN 後端上傳管理器
|
||||
*
|
||||
* 統一處理所有檔案上傳,包含:
|
||||
* - 小檔(≤10MB):直接接收
|
||||
* - 大檔(>10MB):chunk 接收與合併
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants";
|
||||
import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types";
|
||||
import { getTransferMode } from "./types";
|
||||
|
||||
/**
|
||||
* 上傳會話管理器
|
||||
* 用於追蹤進行中的 chunk 上傳
|
||||
*/
|
||||
class UploadSessionManager {
|
||||
private sessions: Map<string, UploadSession> = new Map();
|
||||
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor() {
|
||||
// 定期清理過期的 sessions
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanupExpiredSessions();
|
||||
}, 5 * 60 * 1000); // 每 5 分鐘清理一次
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立新的上傳會話
|
||||
*/
|
||||
createSession(
|
||||
uploadId: string,
|
||||
userId: string,
|
||||
jobId: string,
|
||||
fileName: string,
|
||||
totalSize: number,
|
||||
totalChunks: number,
|
||||
baseTempDir: string
|
||||
): UploadSession {
|
||||
const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId);
|
||||
|
||||
if (!existsSync(tempDir)) {
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
}
|
||||
|
||||
const session: UploadSession = {
|
||||
upload_id: uploadId,
|
||||
user_id: userId,
|
||||
job_id: jobId,
|
||||
file_name: fileName,
|
||||
total_size: totalSize,
|
||||
total_chunks: totalChunks,
|
||||
received_chunks: new Set(),
|
||||
created_at: new Date(),
|
||||
temp_dir: tempDir,
|
||||
};
|
||||
|
||||
this.sessions.set(uploadId, session);
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得上傳會話
|
||||
*/
|
||||
getSession(uploadId: string): UploadSession | undefined {
|
||||
return this.sessions.get(uploadId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新已接收的 chunk
|
||||
*/
|
||||
markChunkReceived(uploadId: string, chunkIndex: number): boolean {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (!session) return false;
|
||||
|
||||
session.received_chunks.add(chunkIndex);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 檢查是否所有 chunks 都已接收
|
||||
*/
|
||||
isComplete(uploadId: string): boolean {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (!session) return false;
|
||||
|
||||
return session.received_chunks.size === session.total_chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除會話並清理暫存檔案
|
||||
*/
|
||||
removeSession(uploadId: string): void {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (session && existsSync(session.temp_dir)) {
|
||||
rmSync(session.temp_dir, { recursive: true, force: true });
|
||||
}
|
||||
this.sessions.delete(uploadId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理過期的會話
|
||||
*/
|
||||
private cleanupExpiredSessions(): void {
|
||||
const now = Date.now();
|
||||
for (const [uploadId, session] of this.sessions) {
|
||||
if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) {
|
||||
console.log(`Cleaning up expired upload session: ${uploadId}`);
|
||||
this.removeSession(uploadId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止清理計時器
|
||||
*/
|
||||
destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 單例實例
|
||||
export const uploadSessionManager = new UploadSessionManager();
|
||||
|
||||
/**
|
||||
* 判斷檔案是否需要使用 chunk 上傳
|
||||
*/
|
||||
export function shouldUseChunkedUpload(fileSize: number): boolean {
|
||||
return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked";
|
||||
}
|
||||
|
||||
/**
|
||||
* 處理直接上傳(小檔)
|
||||
*/
|
||||
export async function handleDirectUpload(
|
||||
file: File,
|
||||
targetDir: string,
|
||||
fileName: string
|
||||
): Promise<DirectUploadResponse> {
|
||||
try {
|
||||
const targetPath = join(targetDir, fileName);
|
||||
|
||||
if (!existsSync(targetDir)) {
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
await Bun.write(targetPath, file);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "File uploaded successfully.",
|
||||
file_path: targetPath,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Direct upload failed:", error);
|
||||
return {
|
||||
success: false,
|
||||
message: `Upload failed: ${error}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 處理 chunk 上傳
|
||||
*/
|
||||
export async function handleChunkUpload(
|
||||
uploadId: string,
|
||||
chunkIndex: number,
|
||||
totalChunks: number,
|
||||
chunkData: ArrayBuffer | Blob | Buffer,
|
||||
fileName: string,
|
||||
totalSize: number,
|
||||
userId: string,
|
||||
jobId: string,
|
||||
baseTempDir: string,
|
||||
targetDir: string
|
||||
): Promise<ChunkUploadResponse> {
|
||||
try {
|
||||
// 取得或建立會話
|
||||
let session = uploadSessionManager.getSession(uploadId);
|
||||
|
||||
if (!session) {
|
||||
session = uploadSessionManager.createSession(
|
||||
uploadId,
|
||||
userId,
|
||||
jobId,
|
||||
fileName,
|
||||
totalSize,
|
||||
totalChunks,
|
||||
baseTempDir
|
||||
);
|
||||
}
|
||||
|
||||
// 驗證會話資訊一致性
|
||||
if (session.total_chunks !== totalChunks || session.file_name !== fileName) {
|
||||
return {
|
||||
success: false,
|
||||
message: "Session mismatch: inconsistent upload parameters",
|
||||
};
|
||||
}
|
||||
|
||||
// 儲存 chunk
|
||||
const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`);
|
||||
// 處理不同類型的 chunk 資料
|
||||
let data: ArrayBuffer | Buffer;
|
||||
if (chunkData instanceof Blob) {
|
||||
data = await chunkData.arrayBuffer();
|
||||
} else if (Buffer.isBuffer(chunkData)) {
|
||||
data = chunkData;
|
||||
} else {
|
||||
data = chunkData;
|
||||
}
|
||||
await Bun.write(chunkPath, data);
|
||||
|
||||
// 標記已接收
|
||||
uploadSessionManager.markChunkReceived(uploadId, chunkIndex);
|
||||
|
||||
// 檢查是否完成
|
||||
if (uploadSessionManager.isComplete(uploadId)) {
|
||||
// 合併所有 chunks
|
||||
const finalPath = await mergeChunks(session, targetDir);
|
||||
|
||||
// 清理會話
|
||||
uploadSessionManager.removeSession(uploadId);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "Upload completed and merged successfully.",
|
||||
received_chunks: Array.from({ length: totalChunks }, (_, i) => i),
|
||||
completed: true,
|
||||
file_path: finalPath,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`,
|
||||
received_chunks: Array.from(session.received_chunks),
|
||||
completed: false,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Chunk upload failed:", error);
|
||||
return {
|
||||
success: false,
|
||||
message: `Chunk upload failed: ${error}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 合併所有 chunks 為完整檔案
|
||||
*/
|
||||
async function mergeChunks(session: UploadSession, targetDir: string): Promise<string> {
|
||||
const targetPath = join(targetDir, session.file_name);
|
||||
|
||||
if (!existsSync(targetDir)) {
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
// 讀取並排序所有 chunks
|
||||
const chunkFiles = readdirSync(session.temp_dir)
|
||||
.filter(f => f.startsWith("chunk_"))
|
||||
.sort();
|
||||
|
||||
// 建立輸出串流
|
||||
const writeStream = createWriteStream(targetPath);
|
||||
|
||||
// 依序寫入每個 chunk
|
||||
for (const chunkFile of chunkFiles) {
|
||||
const chunkPath = join(session.temp_dir, chunkFile);
|
||||
const chunkData = await Bun.file(chunkPath).arrayBuffer();
|
||||
writeStream.write(Buffer.from(chunkData));
|
||||
}
|
||||
|
||||
// 結束寫入
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writeStream.end((err: Error | null) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// 驗證檔案大小
|
||||
const finalStats = statSync(targetPath);
|
||||
if (finalStats.size !== session.total_size) {
|
||||
console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`);
|
||||
}
|
||||
|
||||
console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`);
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 計算需要的 chunk 數量
|
||||
*/
|
||||
export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
return Math.ceil(fileSize / chunkSize);
|
||||
}
|
||||
|
|
@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(mineruArgs).toContain("--table-mode");
|
||||
|
|
@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(capturedArgs).toContain("--table-mode");
|
||||
|
|
@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("MinerU converter tar.gz output", () => {
|
||||
const testDir = "./test-output-mineru-targz";
|
||||
describe("MinerU converter .tar output (no compression)", () => {
|
||||
const testDir = "./test-output-mineru-tar";
|
||||
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
|
|
@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("should create tar.gz archive from output", async () => {
|
||||
test("should create .tar archive from output (without gzip)", async () => {
|
||||
let tarCalled = false;
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
|
|
@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(result).toBe("Done");
|
||||
expect(tarCalled).toBe(true);
|
||||
expect(tarArgs).toContain("-czf");
|
||||
// 驗證使用 -cf 而非 -czf(不壓縮)
|
||||
expect(tarArgs).toContain("-cf");
|
||||
// 確保沒有使用 gzip 壓縮標誌
|
||||
expect(tarArgs).not.toContain("-czf");
|
||||
});
|
||||
|
||||
test("should reject on mineru error", async () => {
|
||||
|
|
@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
|
||||
.rejects.toMatch(/mineru error/);
|
||||
});
|
||||
|
||||
test("should use correct tar arguments for compression", async () => {
|
||||
test("should use correct tar arguments without compression", async () => {
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
|
|
@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// Verify tar is called with correct compression flags
|
||||
expect(tarArgs[0]).toBe("-czf");
|
||||
expect(tarArgs[1]).toContain(".tar.gz");
|
||||
// 驗證 tar 使用正確的無壓縮標誌
|
||||
expect(tarArgs[0]).toBe("-cf");
|
||||
expect(tarArgs[1]).toContain(".tar");
|
||||
expect(tarArgs[1]).not.toContain(".tar.gz");
|
||||
expect(tarArgs[2]).toBe("-C");
|
||||
expect(tarArgs[4]).toBe(".");
|
||||
});
|
||||
|
||||
test("should convert .tar.gz target path to .tar", async () => {
|
||||
let tarOutputPath = "";
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
cmd: string,
|
||||
args: string[],
|
||||
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
|
||||
options?: any,
|
||||
) => {
|
||||
if (cmd === "mineru") {
|
||||
const outputDir = args[3];
|
||||
if (outputDir && !existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
const autoDir = join(outputDir, "auto");
|
||||
if (!existsSync(autoDir)) {
|
||||
mkdirSync(autoDir, { recursive: true });
|
||||
}
|
||||
writeFileSync(join(autoDir, "output.md"), "# Test");
|
||||
callback(null, "Done", "");
|
||||
} else if (cmd === "tar") {
|
||||
tarOutputPath = args[1]; // 輸出路徑
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
// 即使傳入 .tar.gz,也應該被轉換為 .tar
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// 驗證輸出是 .tar 而非 .tar.gz
|
||||
expect(tarOutputPath).toContain(".tar");
|
||||
expect(tarOutputPath).not.toContain(".tar.gz");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
208
tests/transfer/chunk-download.test.ts
Normal file
208
tests/transfer/chunk-download.test.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* Contents.CN Chunk 下載整合測試
|
||||
*
|
||||
* 測試大檔 chunk 下載的完整流程
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
createChunkDownloadHeaders,
|
||||
} from "../../src/transfer/downloadManager";
|
||||
import { CHUNK_SIZE_BYTES, CHUNK_THRESHOLD_BYTES } from "../../src/transfer/constants";
|
||||
|
||||
const testDir = "./test-output-chunk-download";
|
||||
|
||||
describe("Chunk 下載資訊測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("getChunkDownloadInfo 應返回正確的下載資訊", () => {
|
||||
const testFile = join(testDir, "large-file.bin");
|
||||
const fileSize = 25 * 1024 * 1024; // 25MB
|
||||
|
||||
// 建立一個假檔案(只寫入少量資料模擬)
|
||||
const content = Buffer.alloc(1024, "X"); // 1KB
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
|
||||
expect(info).not.toBeNull();
|
||||
expect(info!.file_name).toBe("large-file.bin");
|
||||
expect(info!.total_size).toBe(1024);
|
||||
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
|
||||
expect(info!.total_chunks).toBe(1); // 1KB 只需要 1 chunk
|
||||
});
|
||||
|
||||
test("不存在的檔案應返回 null", () => {
|
||||
const info = getChunkDownloadInfo(join(testDir, "non-existent.bin"));
|
||||
expect(info).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 讀取測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("getChunk 應正確讀取指定的 chunk", async () => {
|
||||
const testFile = join(testDir, "chunked-file.bin");
|
||||
|
||||
// 建立測試檔案:20 bytes,分成 4 個 5-byte chunks
|
||||
const content = "AAAAABBBBBCCCCCDDDD"; // 19 bytes
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
// 讀取第一個 chunk(5 bytes)
|
||||
const chunk0 = await getChunk(testFile, 0, 5);
|
||||
expect(chunk0!.toString()).toBe("AAAAA");
|
||||
|
||||
// 讀取第二個 chunk
|
||||
const chunk1 = await getChunk(testFile, 1, 5);
|
||||
expect(chunk1!.toString()).toBe("BBBBB");
|
||||
|
||||
// 讀取第三個 chunk
|
||||
const chunk2 = await getChunk(testFile, 2, 5);
|
||||
expect(chunk2!.toString()).toBe("CCCCC");
|
||||
|
||||
// 讀取最後一個 chunk(只有 4 bytes)
|
||||
const chunk3 = await getChunk(testFile, 3, 5);
|
||||
expect(chunk3!.toString()).toBe("DDDD");
|
||||
});
|
||||
|
||||
test("讀取超出範圍的 chunk 應返回 null", async () => {
|
||||
const testFile = join(testDir, "small-file.txt");
|
||||
writeFileSync(testFile, "Hello");
|
||||
|
||||
const chunk = await getChunk(testFile, 10, 5); // 超出範圍
|
||||
expect(chunk).toBeNull();
|
||||
});
|
||||
|
||||
test("不存在的檔案應返回 null", async () => {
|
||||
const chunk = await getChunk(join(testDir, "non-existent.bin"), 0, 5);
|
||||
expect(chunk).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 下載 Headers 測試", () => {
|
||||
test("createChunkDownloadHeaders 應生成正確的 headers", () => {
|
||||
const info = {
|
||||
total_size: 1000,
|
||||
total_chunks: 4,
|
||||
chunk_size: 256,
|
||||
file_name: "test-file.bin",
|
||||
};
|
||||
|
||||
const chunkData = Buffer.alloc(256, "X");
|
||||
const headers = createChunkDownloadHeaders(info, 0, chunkData);
|
||||
|
||||
expect(headers["Content-Type"]).toBe("application/octet-stream");
|
||||
expect(headers["Content-Length"]).toBe("256");
|
||||
expect(headers["Content-Range"]).toBe("bytes 0-255/1000");
|
||||
expect(headers["Accept-Ranges"]).toBe("bytes");
|
||||
expect(headers["X-Chunk-Index"]).toBe("0");
|
||||
expect(headers["X-Total-Chunks"]).toBe("4");
|
||||
expect(headers["X-Total-Size"]).toBe("1000");
|
||||
});
|
||||
|
||||
test("最後一個 chunk 的 headers 應正確", () => {
|
||||
const info = {
|
||||
total_size: 1000,
|
||||
total_chunks: 4,
|
||||
chunk_size: 256,
|
||||
file_name: "test-file.bin",
|
||||
};
|
||||
|
||||
// 最後一個 chunk 只有 232 bytes (1000 - 256*3)
|
||||
const chunkData = Buffer.alloc(232, "Z");
|
||||
const headers = createChunkDownloadHeaders(info, 3, chunkData);
|
||||
|
||||
expect(headers["Content-Length"]).toBe("232");
|
||||
expect(headers["Content-Range"]).toBe("bytes 768-999/1000");
|
||||
expect(headers["X-Chunk-Index"]).toBe("3");
|
||||
});
|
||||
});
|
||||
|
||||
describe("傳輸模式判斷測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔(≤10MB)不應使用 chunk 下載", () => {
|
||||
const smallFile = join(testDir, "small.txt");
|
||||
writeFileSync(smallFile, "Small file content");
|
||||
|
||||
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
|
||||
});
|
||||
|
||||
test("不存在的檔案不應使用 chunk 下載", () => {
|
||||
expect(shouldUseChunkedDownload(join(testDir, "non-existent.bin"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("完整 chunk 下載流程測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("應能完整讀取並合併所有 chunks", async () => {
|
||||
const testFile = join(testDir, "complete-file.bin");
|
||||
const originalContent = "AAAAAABBBBBBCCCCCCDDDDDDEEEEEE"; // 30 bytes
|
||||
writeFileSync(testFile, originalContent);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
expect(info).not.toBeNull();
|
||||
|
||||
// 模擬分段讀取(每個 chunk 10 bytes)
|
||||
const chunkSize = 10;
|
||||
const totalChunks = Math.ceil(originalContent.length / chunkSize);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < totalChunks; i++) {
|
||||
const chunk = await getChunk(testFile, i, chunkSize);
|
||||
if (chunk) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
// 合併所有 chunks
|
||||
const mergedContent = Buffer.concat(chunks).toString();
|
||||
expect(mergedContent).toBe(originalContent);
|
||||
});
|
||||
});
|
||||
178
tests/transfer/chunk-upload.test.ts
Normal file
178
tests/transfer/chunk-upload.test.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Contents.CN Chunk 上傳整合測試
|
||||
*
|
||||
* 測試大檔 chunk 上傳的完整流程
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
handleChunkUpload,
|
||||
uploadSessionManager,
|
||||
calculateChunkCount,
|
||||
} from "../../src/transfer/uploadManager";
|
||||
import { CHUNK_SIZE_BYTES } from "../../src/transfer/constants";
|
||||
|
||||
const testDir = "./test-output-chunk-upload";
|
||||
|
||||
describe("Chunk 上傳整合測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("多 chunk 上傳應正確合併", async () => {
|
||||
const uploadId = "test-chunked-upload-1";
|
||||
const fileName = "large-file.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
// 建立測試資料(模擬 15MB 檔案,分 3 個 chunks)
|
||||
const chunkSize = 5 * 1024; // 使用較小的 chunk 以便測試
|
||||
const chunk1 = Buffer.alloc(chunkSize, "A");
|
||||
const chunk2 = Buffer.alloc(chunkSize, "B");
|
||||
const chunk3 = Buffer.alloc(chunkSize, "C");
|
||||
const totalSize = chunkSize * 3;
|
||||
const totalChunks = 3;
|
||||
|
||||
const baseTempDir = join(testDir, "temp");
|
||||
const targetDir = join(testDir, "uploads");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 上傳 chunk 1
|
||||
const result1 = await handleChunkUpload(
|
||||
uploadId, 0, totalChunks, chunk1, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result1.success).toBe(true);
|
||||
expect(result1.completed).toBe(false);
|
||||
expect(result1.received_chunks).toContain(0);
|
||||
|
||||
// 上傳 chunk 2
|
||||
const result2 = await handleChunkUpload(
|
||||
uploadId, 1, totalChunks, chunk2, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result2.success).toBe(true);
|
||||
expect(result2.completed).toBe(false);
|
||||
|
||||
// 上傳 chunk 3(最後一個)
|
||||
const result3 = await handleChunkUpload(
|
||||
uploadId, 2, totalChunks, chunk3, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result3.success).toBe(true);
|
||||
expect(result3.completed).toBe(true);
|
||||
expect(result3.file_path).toBeDefined();
|
||||
|
||||
// 驗證合併後的檔案
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
expect(existsSync(mergedFile)).toBe(true);
|
||||
|
||||
const mergedContent = readFileSync(mergedFile);
|
||||
expect(mergedContent.length).toBe(totalSize);
|
||||
|
||||
// 驗證內容正確性
|
||||
const expectedContent = Buffer.concat([chunk1, chunk2, chunk3]);
|
||||
expect(mergedContent.equals(expectedContent)).toBe(true);
|
||||
});
|
||||
|
||||
test("亂序上傳 chunks 也應正確合併", async () => {
|
||||
const uploadId = "test-chunked-upload-2";
|
||||
const fileName = "out-of-order.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
const chunkSize = 1024;
|
||||
const chunk0 = Buffer.alloc(chunkSize, "0");
|
||||
const chunk1 = Buffer.alloc(chunkSize, "1");
|
||||
const chunk2 = Buffer.alloc(chunkSize, "2");
|
||||
const chunk3 = Buffer.alloc(chunkSize, "3");
|
||||
const totalSize = chunkSize * 4;
|
||||
const totalChunks = 4;
|
||||
|
||||
const baseTempDir = join(testDir, "temp2");
|
||||
const targetDir = join(testDir, "uploads2");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 亂序上傳:3, 1, 0, 2
|
||||
await handleChunkUpload(uploadId, 3, totalChunks, chunk3, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
const finalResult = await handleChunkUpload(uploadId, 2, totalChunks, chunk2, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
expect(finalResult.success).toBe(true);
|
||||
expect(finalResult.completed).toBe(true);
|
||||
|
||||
// 驗證檔案順序正確
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
const mergedContent = readFileSync(mergedFile);
|
||||
|
||||
// 即使亂序上傳,合併後應該按正確順序
|
||||
const expectedContent = Buffer.concat([chunk0, chunk1, chunk2, chunk3]);
|
||||
expect(mergedContent.equals(expectedContent)).toBe(true);
|
||||
});
|
||||
|
||||
test("重複上傳相同 chunk 應被處理", async () => {
|
||||
const uploadId = "test-chunked-upload-3";
|
||||
const fileName = "duplicate-chunk.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
const chunkSize = 512;
|
||||
const chunk0 = Buffer.alloc(chunkSize, "X");
|
||||
const chunk1 = Buffer.alloc(chunkSize, "Y");
|
||||
const totalSize = chunkSize * 2;
|
||||
const totalChunks = 2;
|
||||
|
||||
const baseTempDir = join(testDir, "temp3");
|
||||
const targetDir = join(testDir, "uploads3");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 上傳 chunk 0
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
// 重複上傳 chunk 0
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
// 上傳 chunk 1
|
||||
const result = await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.completed).toBe(true);
|
||||
|
||||
// 驗證檔案大小正確(不應該因為重複上傳而變大)
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
const stats = statSync(mergedFile);
|
||||
expect(stats.size).toBe(totalSize);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 數量計算測試", () => {
|
||||
test("應正確計算不同大小檔案的 chunk 數量", () => {
|
||||
const chunkSize = CHUNK_SIZE_BYTES;
|
||||
|
||||
// 剛好 1 個 chunk
|
||||
expect(calculateChunkCount(chunkSize)).toBe(1);
|
||||
|
||||
// 多 1 byte 需要 2 個 chunks
|
||||
expect(calculateChunkCount(chunkSize + 1)).toBe(2);
|
||||
|
||||
// 50MB 需要 10 個 chunks
|
||||
expect(calculateChunkCount(50 * 1024 * 1024)).toBe(10);
|
||||
|
||||
// 1 byte 需要 1 個 chunk
|
||||
expect(calculateChunkCount(1)).toBe(1);
|
||||
});
|
||||
});
|
||||
331
tests/transfer/transfer.test.ts
Normal file
331
tests/transfer/transfer.test.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸機制測試
|
||||
*
|
||||
* 測試項目:
|
||||
* 1. 小檔直傳測試(≤10MB)
|
||||
* 2. 大檔 chunk 上傳測試(>10MB)
|
||||
* 3. chunk 合併正確性測試
|
||||
* 4. .tar 封裝測試
|
||||
* 5. End-to-End 測試
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
shouldUseChunkedUpload,
|
||||
calculateChunkCount,
|
||||
handleDirectUpload,
|
||||
handleChunkUpload,
|
||||
uploadSessionManager,
|
||||
} from "../../src/transfer/uploadManager";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
} from "../../src/transfer/downloadManager";
|
||||
import {
|
||||
validateArchiveFormat,
|
||||
getArchiveFileName,
|
||||
createTarArchive,
|
||||
} from "../../src/transfer/archiveManager";
|
||||
import {
|
||||
CHUNK_THRESHOLD_BYTES,
|
||||
CHUNK_SIZE_BYTES,
|
||||
ALLOWED_ARCHIVE_FORMAT,
|
||||
FORBIDDEN_ARCHIVE_FORMATS,
|
||||
} from "../../src/transfer/constants";
|
||||
import { getTransferMode } from "../../src/transfer/types";
|
||||
|
||||
const testDir = "./test-output-transfer";
|
||||
|
||||
describe("傳輸常數測試", () => {
|
||||
test("CHUNK_THRESHOLD_BYTES 應為 10MB", () => {
|
||||
expect(CHUNK_THRESHOLD_BYTES).toBe(10 * 1024 * 1024);
|
||||
});
|
||||
|
||||
test("CHUNK_SIZE_BYTES 應為 5MB", () => {
|
||||
expect(CHUNK_SIZE_BYTES).toBe(5 * 1024 * 1024);
|
||||
});
|
||||
|
||||
test("唯一允許的封裝格式應為 .tar", () => {
|
||||
expect(ALLOWED_ARCHIVE_FORMAT).toBe(".tar");
|
||||
});
|
||||
|
||||
test("禁止的封裝格式應包含 .tar.gz, .tgz, .zip, .gz", () => {
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tar.gz");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tgz");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".zip");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".gz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("傳輸模式判斷測試", () => {
|
||||
test("≤10MB 應使用直傳模式", () => {
|
||||
expect(getTransferMode(10 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(5 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(0, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
});
|
||||
|
||||
test(">10MB 應使用 chunk 模式", () => {
|
||||
expect(getTransferMode(10 * 1024 * 1024 + 1, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
expect(getTransferMode(50 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
expect(getTransferMode(100 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
});
|
||||
|
||||
test("shouldUseChunkedUpload 函數應正確判斷", () => {
|
||||
expect(shouldUseChunkedUpload(10 * 1024 * 1024)).toBe(false);
|
||||
expect(shouldUseChunkedUpload(10 * 1024 * 1024 + 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 數量計算測試", () => {
|
||||
test("計算 chunk 數量應正確", () => {
|
||||
// 10MB / 5MB = 2 chunks
|
||||
expect(calculateChunkCount(10 * 1024 * 1024)).toBe(2);
|
||||
|
||||
// 15MB / 5MB = 3 chunks
|
||||
expect(calculateChunkCount(15 * 1024 * 1024)).toBe(3);
|
||||
|
||||
// 1 byte 也需要 1 chunk
|
||||
expect(calculateChunkCount(1)).toBe(1);
|
||||
|
||||
// 5MB + 1 byte = 2 chunks
|
||||
expect(calculateChunkCount(5 * 1024 * 1024 + 1)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("封裝格式驗證測試", () => {
|
||||
test("validateArchiveFormat 應正確驗證格式", () => {
|
||||
// 允許的格式
|
||||
expect(validateArchiveFormat("output.tar")).toBe(true);
|
||||
expect(validateArchiveFormat("my_archive.tar")).toBe(true);
|
||||
|
||||
// 禁止的格式
|
||||
expect(validateArchiveFormat("output.tar.gz")).toBe(false);
|
||||
expect(validateArchiveFormat("output.tgz")).toBe(false);
|
||||
expect(validateArchiveFormat("output.zip")).toBe(false);
|
||||
expect(validateArchiveFormat("output.gz")).toBe(false);
|
||||
});
|
||||
|
||||
test("getArchiveFileName 應強制轉換為 .tar", () => {
|
||||
// 已經是 .tar
|
||||
expect(getArchiveFileName("output.tar")).toBe("output.tar");
|
||||
|
||||
// 轉換 .tar.gz -> .tar
|
||||
expect(getArchiveFileName("output.tar.gz")).toBe("output.tar");
|
||||
|
||||
// 轉換 .tgz -> .tar
|
||||
expect(getArchiveFileName("output.tgz")).toBe("output.tar");
|
||||
|
||||
// 無副檔名 -> 加上 .tar
|
||||
expect(getArchiveFileName("output")).toBe("output.tar");
|
||||
|
||||
// .zip 也是禁止的格式,會被移除並加上 .tar
|
||||
expect(getArchiveFileName("output.zip")).toBe("output.tar");
|
||||
|
||||
// .gz 也是禁止的格式
|
||||
expect(getArchiveFileName("output.gz")).toBe("output.tar");
|
||||
});
|
||||
});
|
||||
|
||||
describe("上傳管理測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔直傳應成功", async () => {
|
||||
const targetDir = join(testDir, "uploads");
|
||||
const testContent = "Hello, World!";
|
||||
const file = new File([testContent], "test.txt", { type: "text/plain" });
|
||||
|
||||
const result = await handleDirectUpload(file as any, targetDir, "test.txt");
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(existsSync(join(targetDir, "test.txt"))).toBe(true);
|
||||
expect(readFileSync(join(targetDir, "test.txt"), "utf-8")).toBe(testContent);
|
||||
});
|
||||
});
|
||||
|
||||
describe("下載管理測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔不應使用 chunk 下載", () => {
|
||||
const smallFile = join(testDir, "small.txt");
|
||||
writeFileSync(smallFile, "Small content");
|
||||
|
||||
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
|
||||
});
|
||||
|
||||
test("getChunkDownloadInfo 應返回正確資訊", () => {
|
||||
const testFile = join(testDir, "test.txt");
|
||||
const content = "Test content for download";
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
|
||||
expect(info).not.toBeNull();
|
||||
expect(info!.file_name).toBe("test.txt");
|
||||
expect(info!.total_size).toBe(content.length);
|
||||
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
|
||||
});
|
||||
|
||||
test("getChunk 應正確讀取 chunk 資料", async () => {
|
||||
const testFile = join(testDir, "test.txt");
|
||||
const content = "ABCDEFGHIJ"; // 10 bytes
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
// 讀取第一個 chunk(整個檔案,因為小於 chunk size)
|
||||
const chunk = await getChunk(testFile, 0, 5);
|
||||
|
||||
expect(chunk).not.toBeNull();
|
||||
expect(chunk!.toString()).toBe("ABCDE");
|
||||
|
||||
// 讀取第二個 chunk
|
||||
const chunk2 = await getChunk(testFile, 1, 5);
|
||||
expect(chunk2!.toString()).toBe("FGHIJ");
|
||||
});
|
||||
});
|
||||
|
||||
describe(".tar 封裝測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("createTarArchive 應建立 .tar 檔案", async () => {
|
||||
const sourceDir = join(testDir, "source");
|
||||
const outputTar = join(testDir, "output.tar");
|
||||
|
||||
mkdirSync(sourceDir, { recursive: true });
|
||||
writeFileSync(join(sourceDir, "file1.txt"), "Content 1");
|
||||
writeFileSync(join(sourceDir, "file2.txt"), "Content 2");
|
||||
|
||||
const result = await createTarArchive(sourceDir, outputTar);
|
||||
|
||||
expect(existsSync(result)).toBe(true);
|
||||
expect(result.endsWith(".tar")).toBe(true);
|
||||
expect(result).not.toContain(".tar.gz");
|
||||
});
|
||||
|
||||
test("createTarArchive 應強制使用 .tar 格式", async () => {
|
||||
const sourceDir = join(testDir, "source2");
|
||||
// 即使傳入 .tar.gz,也應該輸出 .tar
|
||||
const outputTar = join(testDir, "output.tar.gz");
|
||||
|
||||
mkdirSync(sourceDir, { recursive: true });
|
||||
writeFileSync(join(sourceDir, "file.txt"), "Content");
|
||||
|
||||
const result = await createTarArchive(sourceDir, outputTar);
|
||||
|
||||
// 結果應該是 .tar 而非 .tar.gz
|
||||
expect(result.endsWith(".tar")).toBe(true);
|
||||
expect(result).not.toContain(".tar.gz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("End-to-End 測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("完整流程:上傳 -> 封裝 -> 下載資訊", async () => {
|
||||
// 1. 模擬上傳
|
||||
const uploadsDir = join(testDir, "uploads");
|
||||
const outputDir = join(testDir, "output");
|
||||
mkdirSync(uploadsDir, { recursive: true });
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const uploadedFile = join(uploadsDir, "document.pdf");
|
||||
writeFileSync(uploadedFile, "Fake PDF content");
|
||||
|
||||
// 2. 模擬轉換輸出
|
||||
const conversionOutput = join(outputDir, "conversion_result");
|
||||
mkdirSync(conversionOutput, { recursive: true });
|
||||
writeFileSync(join(conversionOutput, "output.md"), "# Converted Document");
|
||||
mkdirSync(join(conversionOutput, "images"), { recursive: true });
|
||||
writeFileSync(join(conversionOutput, "images", "fig1.png"), "Fake image");
|
||||
|
||||
// 3. 建立 .tar 封裝
|
||||
const tarPath = join(outputDir, "result.tar");
|
||||
const archivePath = await createTarArchive(conversionOutput, tarPath);
|
||||
|
||||
expect(existsSync(archivePath)).toBe(true);
|
||||
expect(archivePath.endsWith(".tar")).toBe(true);
|
||||
|
||||
// 4. 取得下載資訊
|
||||
const downloadInfo = getChunkDownloadInfo(archivePath);
|
||||
expect(downloadInfo).not.toBeNull();
|
||||
expect(downloadInfo!.file_name).toBe("result.tar");
|
||||
|
||||
// 5. 驗證傳輸模式判斷
|
||||
const fileStats = statSync(archivePath);
|
||||
const shouldChunk = fileStats.size > CHUNK_THRESHOLD_BYTES;
|
||||
expect(shouldUseChunkedDownload(archivePath)).toBe(shouldChunk);
|
||||
});
|
||||
});
|
||||
|
||||
describe("上傳會話管理測試", () => {
|
||||
test("應正確建立和管理上傳會話", () => {
|
||||
const uploadId = "test-upload-123";
|
||||
const session = uploadSessionManager.createSession(
|
||||
uploadId,
|
||||
"user-1",
|
||||
"job-1",
|
||||
"large-file.pdf",
|
||||
50 * 1024 * 1024, // 50MB
|
||||
10, // 10 chunks
|
||||
testDir
|
||||
);
|
||||
|
||||
expect(session.upload_id).toBe(uploadId);
|
||||
expect(session.total_chunks).toBe(10);
|
||||
expect(session.received_chunks.size).toBe(0);
|
||||
|
||||
// 標記已接收 chunks
|
||||
uploadSessionManager.markChunkReceived(uploadId, 0);
|
||||
uploadSessionManager.markChunkReceived(uploadId, 1);
|
||||
|
||||
const updatedSession = uploadSessionManager.getSession(uploadId);
|
||||
expect(updatedSession!.received_chunks.size).toBe(2);
|
||||
expect(uploadSessionManager.isComplete(uploadId)).toBe(false);
|
||||
|
||||
// 清理
|
||||
uploadSessionManager.removeSession(uploadId);
|
||||
expect(uploadSessionManager.getSession(uploadId)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue