From 5322f85721e76554c528ae80688c8c12a9739b1d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 21 Jan 2026 13:58:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AF=A6=E4=BD=9C=E5=85=A8=E5=9F=9F?= =?UTF-8?q?=E6=AA=94=E6=A1=88=E5=82=B3=E8=BC=B8=E6=A9=9F=E5=88=B6=E8=88=87?= =?UTF-8?q?=20.tar=20=E5=B0=81=E8=A3=9D=E8=A6=8F=E7=AF=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數（10MB 門檻、5MB chunk 大小） - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器（支援直傳與 chunk） - downloadManager.ts: 後端下載管理器（支援直傳與 chunk） - archiveManager.ts: 封裝管理器（僅允許 .tar） - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略（≤10MB 直傳，>10MB chunk） - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式（不壓縮），禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB：直接傳輸 - 檔案 >10MB：使用 5MB chunks 分段傳輸 - 多檔輸出：僅允許 .tar 封裝（禁止 .tar.gz/.zip） - 引擎層不感知 chunk（僅傳輸層處理） --- README.md | 46 ++- public/script.js | 134 ++++++++- public/transfer.js | 388 ++++++++++++++++++++++++++ src/converters/mineru.ts | 33 ++- src/index.tsx | 5 + src/pages/download.tsx | 18 +- src/pages/downloadChunk.tsx | 178 ++++++++++++ src/pages/uploadChunk.tsx | 88 ++++++ src/transfer/archiveManager.ts | 154 ++++++++++ src/transfer/constants.ts | 38 +++ src/transfer/downloadManager.ts | 123 ++++++++ src/transfer/index.ts | 56 ++++ src/transfer/types.ts | 94 +++++++ src/transfer/uploadManager.ts | 303 ++++++++++++++++++++ tests/converters/mineru.test.ts | 73 ++++- tests/transfer/chunk-download.test.ts | 208 ++++++++++++++ tests/transfer/chunk-upload.test.ts | 178 ++++++++++++ tests/transfer/transfer.test.ts | 331 ++++++++++++++++++++++ 18 files changed, 2387 insertions(+), 61 deletions(-) create mode 100644 public/transfer.js create mode 100644 src/pages/downloadChunk.tsx create mode 100644 src/pages/uploadChunk.tsx create mode 100644 src/transfer/archiveManager.ts create mode 100644 src/transfer/constants.ts create mode 100644 src/transfer/downloadManager.ts create mode 100644 src/transfer/index.ts create mode 100644 src/transfer/types.ts create mode 100644 src/transfer/uploadManager.ts create mode 100644 tests/transfer/chunk-download.test.ts create mode 100644 tests/transfer/chunk-upload.test.ts create mode 100644 tests/transfer/transfer.test.ts diff --git a/README.md b/README.md index 3d20689..cfd4271 100644 --- a/README.md +++ b/README.md @@ -157,15 +157,15 @@ docker compose up -d ## 支援格式 -| 轉換器 | 用途 | 格式數 | -| ----------- | ------ | ------ | -| FFmpeg | 影音 | 400+ | -| ImageMagick | 圖片 | 200+ | -| LibreOffice | 文件 | 60+ | -| Pandoc | 文件 | 100+ | -| Calibre | 電子書 | 40+ | -| Inkscape | 向量圖 | 20+ | -| MinerU | 文件→MD | 2 | +| 轉換器 | 用途 | 格式數 | +| ----------- | ------- | ------ | +| FFmpeg | 影音 | 400+ | +| ImageMagick | 圖片 | 200+ | +| LibreOffice | 文件 | 60+ | +| Pandoc | 文件 | 100+ | +| Calibre | 電子書 | 40+ | +| Inkscape | 向量圖 | 20+ | +| MinerU | 文件→MD | 2 | 完整列表 → [docs/converters.md](docs/converters.md) @@ -175,10 +175,32 @@ docker compose up -d ConvertX 內建文件轉換引擎。 -- md-t -- md-i +- md-t（表格以 Markdown 呈現） +- md-i（表格以圖片呈現） -輸出格式：tar.gz +輸出格式：.tar（不壓縮封裝） + +--- + +## 檔案傳輸機制 + +Contents.CN 使用統一的檔案傳輸策略： + +| 檔案大小 | 傳輸方式 | 說明 | +| -------- | -------- | ---- | +| ≤ 10MB | 直接傳輸 | 單一請求完成上傳/下載 | +| > 10MB | 分段傳輸 | 使用 5MB chunks 分段傳輸 | + +### 多檔輸出封裝 + +- ✅ 唯一允許格式：`.tar` +- ❌ 禁止使用：`.tar.gz`、`.tgz`、`.zip` + +### 設計原則 + +- 分段傳輸僅存在於傳輸層 +- 轉換引擎只接收完整檔案 +- 前後端使用共用傳輸模組 --- diff --git a/public/script.js b/public/script.js index aaac8a9..837d029 100644 --- a/public/script.js +++ b/public/script.js @@ -242,11 +242,54 @@ const deleteRow = (target) => { }).catch((err) => console.log(err)); }; +// ==================== 全域傳輸常數 ==================== +const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB +const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB + +/** + * 判斷是否需要使用 chunk 傳輸 + */ +function shouldUseChunkedUpload(fileSize) { + return fileSize > CHUNK_THRESHOLD_BYTES; +} + +/** + * 生成 UUID + */ +function generateUploadId() { + if (typeof crypto !== "undefined" && crypto.randomUUID) { + return crypto.randomUUID(); + } + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c === "x" ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +} + +/** + * 統一上傳檔案（自動判斷使用直傳或 chunk） + * + * @param {File} file - 要上傳的檔案 + */ const uploadFile = (file) => { convertButton.disabled = true; convertButton.value = getTranslation("convert", "uploading"); pendingFiles += 1; + if (shouldUseChunkedUpload(file.size)) { + // 大檔：使用 chunk 上傳 + uploadFileChunked(file); + } else { + // 小檔：直接上傳 + uploadFileDirect(file); + } +}; + +/** + * 直接上傳（≤10MB） + */ +const uploadFileDirect = (file) => { const formData = new FormData(); formData.append("file", file, file.name); @@ -255,7 +298,12 @@ const uploadFile = (file) => { xhr.open("POST", `${webroot}/upload`, true); xhr.onload = () => { - let data = JSON.parse(xhr.responseText); + let data = {}; + try { + data = JSON.parse(xhr.responseText); + } catch (e) { + console.log("Parse error:", e); + } pendingFiles -= 1; if (pendingFiles === 0) { @@ -265,10 +313,12 @@ const uploadFile = (file) => { convertButton.value = getTranslation("convert", "convertButton"); } - //Remove the progress bar when upload is done + // Remove the progress bar when upload is done let progressbar = file.htmlRow.getElementsByTagName("progress"); - progressbar[0].parentElement.remove(); - console.log(data); + if (progressbar[0]) { + progressbar[0].parentElement.remove(); + } + console.log("Direct upload complete:", data); }; xhr.upload.onprogress = (e) => { @@ -277,16 +327,88 @@ const uploadFile = (file) => { console.log(`upload progress (${file.name}):`, (100 * sent) / total); let progressbar = file.htmlRow.getElementsByTagName("progress"); - progressbar[0].value = (100 * sent) / total; + if (progressbar[0]) { + progressbar[0].value = (100 * sent) / total; + } }; xhr.onerror = (e) => { - console.log(e); + console.log("Upload error:", e); + pendingFiles -= 1; + if (pendingFiles === 0) { + convertButton.value = getTranslation("convert", "convertButton"); + } }; xhr.send(formData); }; +/** + * Chunk 上傳（>10MB） + */ +const uploadFileChunked = async (file) => { + const uploadId = generateUploadId(); + const totalChunks = Math.ceil(file.size / CHUNK_SIZE_BYTES); + + console.log(`Starting chunked upload: ${file.name}, size: ${file.size}, chunks: ${totalChunks}`); + + try { + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const start = chunkIndex * CHUNK_SIZE_BYTES; + const end = Math.min(start + CHUNK_SIZE_BYTES, file.size); + const chunk = file.slice(start, end); + + const formData = new FormData(); + formData.append("upload_id", uploadId); + formData.append("chunk_index", chunkIndex.toString()); + formData.append("total_chunks", totalChunks.toString()); + formData.append("file_name", file.name); + formData.append("total_size", file.size.toString()); + formData.append("chunk", chunk); + + const response = await fetch(`${webroot}/upload-chunk`, { + method: "POST", + body: formData, + }); + + if (!response.ok) { + throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`); + } + + // 更新進度 + const percent = ((chunkIndex + 1) / totalChunks) * 100; + let progressbar = file.htmlRow.getElementsByTagName("progress"); + if (progressbar[0]) { + progressbar[0].value = percent; + } + console.log(`Chunk ${chunkIndex + 1}/${totalChunks} uploaded (${percent.toFixed(1)}%)`); + } + + // 完成 + pendingFiles -= 1; + if (pendingFiles === 0) { + if (formatSelected) { + convertButton.disabled = false; + } + convertButton.value = getTranslation("convert", "convertButton"); + } + + // Remove the progress bar + let progressbar = file.htmlRow.getElementsByTagName("progress"); + if (progressbar[0]) { + progressbar[0].parentElement.remove(); + } + console.log("Chunked upload complete:", file.name); + + } catch (error) { + console.error("Chunked upload failed:", error); + pendingFiles -= 1; + if (pendingFiles === 0) { + convertButton.value = getTranslation("convert", "convertButton"); + } + } +}; + const formConvert = document.querySelector(`form[action='${webroot}/convert']`); formConvert.addEventListener("submit", () => { diff --git a/public/transfer.js b/public/transfer.js new file mode 100644 index 0000000..da9405d --- /dev/null +++ b/public/transfer.js @@ -0,0 +1,388 @@ +/** + * Contents.CN 前端檔案傳輸管理器 + * + * 統一處理所有檔案的上傳與下載： + * - 檔案 ≤ 10MB：直接傳輸 + * - 檔案 > 10MB：使用 chunk 分段傳輸 + * + * ⚠️ 重要：所有功能必須使用此模組，不得自行實作傳輸邏輯 + */ + +// ==================== 常數定義 ==================== + +/** + * 檔案大小門檻（10MB） + */ +const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; + +/** + * 每個 chunk 的大小（5MB） + */ +const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; + +// ==================== 工具函數 ==================== + +/** + * 生成 UUID（用於 upload_id） + */ +function generateUploadId() { + if (typeof crypto !== "undefined" && crypto.randomUUID) { + return crypto.randomUUID(); + } + // 降級方案 + return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) { + const r = Math.random() * 16 | 0; + const v = c === "x" ? r : (r & 0x3 | 0x8); + return v.toString(16); + }); +} + +/** + * 判斷是否需要使用 chunk 傳輸 + */ +function shouldUseChunkedTransfer(fileSize) { + return fileSize > CHUNK_THRESHOLD_BYTES; +} + +/** + * 計算 chunk 數量 + */ +function calculateChunkCount(fileSize) { + return Math.ceil(fileSize / CHUNK_SIZE_BYTES); +} + +// ==================== 上傳管理器 ==================== + +/** + * 上傳管理器類別 + */ +class UploadManager { + /** + * @param {string} webroot - 網站根路徑 + */ + constructor(webroot) { + this.webroot = webroot; + this.activeUploads = new Map(); + } + + /** + * 上傳檔案（自動判斷使用直傳或 chunk） + * + * @param {File} file - 要上傳的檔案 + * @param {object} options - 選項 + * @param {function} options.onProgress - 進度回調 (percent: number) => void + * @param {function} options.onComplete - 完成回調 (response: object) => void + * @param {function} options.onError - 錯誤回調 (error: Error) => void + * @returns {Promise} 上傳結果 + */ + async uploadFile(file, options = {}) { + const { onProgress, onComplete, onError } = options; + + try { + let result; + + if (shouldUseChunkedTransfer(file.size)) { + // 大檔：使用 chunk 上傳 + result = await this.uploadChunked(file, onProgress); + } else { + // 小檔：直接上傳 + result = await this.uploadDirect(file, onProgress); + } + + if (onComplete) onComplete(result); + return result; + } catch (error) { + if (onError) onError(error); + throw error; + } + } + + /** + * 直接上傳（小檔） + */ + async uploadDirect(file, onProgress) { + return new Promise((resolve, reject) => { + const formData = new FormData(); + formData.append("file", file, file.name); + + const xhr = new XMLHttpRequest(); + xhr.open("POST", `${this.webroot}/upload`, true); + + xhr.upload.onprogress = (e) => { + if (e.lengthComputable && onProgress) { + const percent = (e.loaded / e.total) * 100; + onProgress(percent); + } + }; + + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) { + try { + const data = JSON.parse(xhr.responseText); + resolve(data); + } catch { + resolve({ success: true, message: "Upload completed" }); + } + } else { + reject(new Error(`Upload failed with status ${xhr.status}`)); + } + }; + + xhr.onerror = () => reject(new Error("Upload failed")); + xhr.send(formData); + }); + } + + /** + * Chunk 上傳（大檔） + */ + async uploadChunked(file, onProgress) { + const uploadId = generateUploadId(); + const totalChunks = calculateChunkCount(file.size); + + this.activeUploads.set(uploadId, { + file, + totalChunks, + uploadedChunks: 0, + status: "uploading" + }); + + try { + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const start = chunkIndex * CHUNK_SIZE_BYTES; + const end = Math.min(start + CHUNK_SIZE_BYTES, file.size); + const chunk = file.slice(start, end); + + await this.uploadChunk(uploadId, chunkIndex, totalChunks, chunk, file.name, file.size); + + // 更新進度 + const uploadInfo = this.activeUploads.get(uploadId); + if (uploadInfo) { + uploadInfo.uploadedChunks = chunkIndex + 1; + const percent = ((chunkIndex + 1) / totalChunks) * 100; + if (onProgress) onProgress(percent); + } + } + + this.activeUploads.delete(uploadId); + return { success: true, message: "Chunked upload completed", upload_id: uploadId }; + } catch (error) { + this.activeUploads.delete(uploadId); + throw error; + } + } + + /** + * 上傳單個 chunk + */ + async uploadChunk(uploadId, chunkIndex, totalChunks, chunkData, fileName, totalSize) { + const formData = new FormData(); + formData.append("upload_id", uploadId); + formData.append("chunk_index", chunkIndex.toString()); + formData.append("total_chunks", totalChunks.toString()); + formData.append("file_name", fileName); + formData.append("total_size", totalSize.toString()); + formData.append("chunk", chunkData); + + const response = await fetch(`${this.webroot}/upload-chunk`, { + method: "POST", + body: formData, + }); + + if (!response.ok) { + throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`); + } + + return response.json(); + } + + /** + * 取消上傳 + */ + cancelUpload(uploadId) { + const upload = this.activeUploads.get(uploadId); + if (upload) { + upload.status = "cancelled"; + this.activeUploads.delete(uploadId); + } + } +} + +// ==================== 下載管理器 ==================== + +/** + * 下載管理器類別 + */ +class DownloadManager { + /** + * @param {string} webroot - 網站根路徑 + */ + constructor(webroot) { + this.webroot = webroot; + } + + /** + * 下載檔案（自動判斷使用直傳或 chunk） + * + * @param {string} url - 下載 URL + * @param {string} fileName - 檔案名稱 + * @param {object} options - 選項 + * @param {function} options.onProgress - 進度回調 + * @returns {Promise} 下載的檔案 + */ + async downloadFile(url, fileName, options = {}) { + const { onProgress } = options; + + // 先取得檔案資訊 + const info = await this.getFileInfo(url); + + if (!info) { + // 無法取得資訊，使用直接下載 + return this.downloadDirect(url, fileName); + } + + if (shouldUseChunkedTransfer(info.total_size)) { + // 大檔：使用 chunk 下載 + return this.downloadChunked(url, fileName, info, onProgress); + } else { + // 小檔：直接下載 + return this.downloadDirect(url, fileName, onProgress); + } + } + + /** + * 取得檔案資訊 + */ + async getFileInfo(url) { + try { + const response = await fetch(`${url}/info`, { method: "GET" }); + if (response.ok) { + return response.json(); + } + } catch { + // 忽略錯誤，降級為直接下載 + } + return null; + } + + /** + * 直接下載（小檔） + */ + async downloadDirect(url, fileName, onProgress) { + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Download failed: ${response.status}`); + } + + const contentLength = response.headers.get("Content-Length"); + const total = contentLength ? parseInt(contentLength, 10) : 0; + + const reader = response.body.getReader(); + const chunks = []; + let loaded = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + chunks.push(value); + loaded += value.length; + + if (onProgress && total > 0) { + onProgress((loaded / total) * 100); + } + } + + const blob = new Blob(chunks); + this.triggerDownload(blob, fileName); + return blob; + } + + /** + * Chunk 下載（大檔） + */ + async downloadChunked(url, fileName, info, onProgress) { + const { total_chunks, chunk_size, total_size } = info; + const chunks = []; + + for (let i = 0; i < total_chunks; i++) { + const chunkData = await this.downloadChunk(url, i); + chunks.push(chunkData); + + if (onProgress) { + const loaded = Math.min((i + 1) * chunk_size, total_size); + onProgress((loaded / total_size) * 100); + } + } + + const blob = new Blob(chunks); + this.triggerDownload(blob, fileName); + return blob; + } + + /** + * 下載單個 chunk + */ + async downloadChunk(url, chunkIndex) { + const response = await fetch(`${url}/chunk/${chunkIndex}`); + + if (!response.ok) { + throw new Error(`Chunk ${chunkIndex} download failed: ${response.status}`); + } + + return response.arrayBuffer(); + } + + /** + * 觸發瀏覽器下載 + */ + triggerDownload(blob, fileName) { + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + + /** + * 簡單下載（不使用 chunk，用於向後相容） + */ + async simpleDownload(url, fileName) { + const a = document.createElement("a"); + a.href = url; + a.download = fileName; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + } +} + +// ==================== 全域實例 ==================== + +// 從 meta 標籤取得 webroot +const webrootMeta = document.querySelector("meta[name='webroot']"); +const webroot = webrootMeta ? webrootMeta.content : ""; + +// 建立全域實例 +window.ContentsTransfer = { + uploadManager: new UploadManager(webroot), + downloadManager: new DownloadManager(webroot), + + // 工具函數 + shouldUseChunkedTransfer, + calculateChunkCount, + + // 常數 + CHUNK_THRESHOLD_BYTES, + CHUNK_SIZE_BYTES, +}; + +// 向後相容：提供簡化的 API +window.uploadFile = (file, options) => window.ContentsTransfer.uploadManager.uploadFile(file, options); +window.downloadFile = (url, fileName, options) => window.ContentsTransfer.downloadManager.downloadFile(url, fileName, options); + +console.log("Contents.CN Transfer Module initialized"); diff --git a/src/converters/mineru.ts b/src/converters/mineru.ts index ce81d8f..1438c0e 100644 --- a/src/converters/mineru.ts +++ b/src/converters/mineru.ts @@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process"; import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs"; import { join, basename, dirname } from "node:path"; import { ExecFileFn } from "./types"; +import { createConverterArchive, getArchiveFileName } from "../transfer"; export const properties = { from: { @@ -14,19 +15,22 @@ export const properties = { }; /** - * Helper function to create a tar.gz archive from a directory + * Helper function to create a .tar archive from a directory (no compression) + * + * ⚠️ 重要：僅使用 .tar 格式，禁止 .tar.gz / .tgz / .zip */ -function createTarGzArchive( +function createTarArchive( sourceDir: string, - outputTarGz: string, + outputTar: string, execFile: ExecFileFn, ): Promise { return new Promise((resolve, reject) => { - // Use tar command to create gzipped archive - // tar -czf -C . + // Use tar command to create archive (without gzip compression) + // tar -cf -C . + // 注意：使用 -cf 而非 -czf，避免 gzip 壓縮 execFile( "tar", - ["-czf", outputTarGz, "-C", sourceDir, "."], + ["-cf", outputTar, "-C", sourceDir, "."], (error, stdout, stderr) => { if (error) { reject(`tar error: ${error}`); @@ -122,15 +126,14 @@ export async function convert( // MinerU outputs to a subdirectory, find the actual output const mineruActualOutput = join(mineruOutputDir, "auto"); - // Create tar.gz archive from the output directory - const tarGzPath = targetPath.endsWith(".tar.gz") - ? targetPath - : `${targetPath}.tar.gz`; + // Create .tar archive from the output directory (不使用壓縮) + // 強制使用 .tar 格式，禁止 .tar.gz + const tarPath = getArchiveFileName(targetPath); // Ensure the parent directory exists - const tarGzDir = dirname(tarGzPath); - if (!existsSync(tarGzDir)) { - mkdirSync(tarGzDir, { recursive: true }); + const tarDir = dirname(tarPath); + if (!existsSync(tarDir)) { + mkdirSync(tarDir, { recursive: true }); } // Use the actual MinerU output directory for archiving @@ -138,14 +141,14 @@ export async function convert( ? mineruActualOutput : mineruOutputDir; - await createTarGzArchive(outputToArchive, tarGzPath, execFile); + await createTarArchive(outputToArchive, tarPath, execFile); // Clean up the temporary directory removeDir(mineruOutputDir); resolve("Done"); } catch (tarError) { - reject(`Failed to create tar.gz archive: ${tarError}`); + reject(`Failed to create .tar archive: ${tarError}`); } }); }); diff --git a/src/index.tsx b/src/index.tsx index c163e74..d5a7b4b 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -11,11 +11,13 @@ import { convert } from "./pages/convert"; import { deleteFile } from "./pages/deleteFile"; import { deleteJob } from "./pages/deleteJob"; import { download } from "./pages/download"; +import { downloadChunk } from "./pages/downloadChunk"; import { history } from "./pages/history"; import { listConverters } from "./pages/listConverters"; import { results } from "./pages/results"; import { root } from "./pages/root"; import { upload } from "./pages/upload"; +import { uploadChunk, uploadInfo } from "./pages/uploadChunk"; import { user } from "./pages/user"; import { healthcheck } from "./pages/healthcheck"; @@ -41,9 +43,12 @@ const app = new Elysia({ .use(user) .use(root) .use(upload) + .use(uploadChunk) + .use(uploadInfo) .use(history) .use(convert) .use(download) + .use(downloadChunk) .use(deleteJob) .use(results) .use(deleteFile) diff --git a/src/pages/download.tsx b/src/pages/download.tsx index f10fec4..9701957 100644 --- a/src/pages/download.tsx +++ b/src/pages/download.tsx @@ -1,11 +1,11 @@ import path from "node:path"; import { Elysia } from "elysia"; import sanitize from "sanitize-filename"; -import * as tar from "tar"; import { outputDir } from ".."; import db from "../db/db"; import { WEBROOT } from "../helpers/env"; import { userService } from "./user"; +import { createJobArchive, shouldUseChunkedDownload } from "../transfer"; export const download = new Elysia() .use(userService) @@ -45,18 +45,10 @@ export const download = new Elysia() const jobId = decodeURIComponent(params.jobId); const outputPath = `${outputDir}${userId}/${jobId}`; - const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`); - - await tar.create( - { - file: outputTar, - cwd: outputPath, - filter: (path) => { - return !path.match(".*\\.tar"); - }, - }, - ["."], - ); + + // 使用統一的封裝管理器建立 .tar（不壓縮） + const outputTar = await createJobArchive(outputPath, jobId); + return Bun.file(outputTar); }, { diff --git a/src/pages/downloadChunk.tsx b/src/pages/downloadChunk.tsx new file mode 100644 index 0000000..8ccc358 --- /dev/null +++ b/src/pages/downloadChunk.tsx @@ -0,0 +1,178 @@ +/** + * Contents.CN Chunk 下載 API + * + * 處理大檔的分段下載 + */ + +import { Elysia, t } from "elysia"; +import { join } from "node:path"; +import { outputDir } from ".."; +import db from "../db/db"; +import { WEBROOT } from "../helpers/env"; +import { userService } from "./user"; +import sanitize from "sanitize-filename"; +import { + shouldUseChunkedDownload, + getChunkDownloadInfo, + getChunk, + createChunkDownloadHeaders, + getFileForDirectDownload, + createDirectDownloadHeaders, + CHUNK_SIZE_BYTES, +} from "../transfer"; +import { statSync, existsSync } from "node:fs"; + +export const downloadChunk = new Elysia() + .use(userService) + /** + * 取得檔案下載資訊 + */ + .get( + "/download/:userId/:jobId/:fileName/info", + async ({ params, user }) => { + const userId = user.id; + const job = await db + .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") + .get(user.id, params.jobId); + + if (!job) { + return { error: "Job not found" }; + } + + const jobId = decodeURIComponent(params.jobId); + const fileName = sanitize(decodeURIComponent(params.fileName)); + const filePath = `${outputDir}${userId}/${jobId}/${fileName}`; + + const info = getChunkDownloadInfo(filePath); + + if (!info) { + return { error: "File not found" }; + } + + return { + ...info, + use_chunked: shouldUseChunkedDownload(filePath), + }; + }, + { auth: true } + ) + /** + * 下載特定 chunk + */ + .get( + "/download/:userId/:jobId/:fileName/chunk/:chunkIndex", + async ({ params, set, user }) => { + const userId = user.id; + const job = await db + .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") + .get(user.id, params.jobId); + + if (!job) { + set.status = 404; + return { error: "Job not found" }; + } + + const jobId = decodeURIComponent(params.jobId); + const fileName = sanitize(decodeURIComponent(params.fileName)); + const chunkIndex = parseInt(params.chunkIndex, 10); + const filePath = `${outputDir}${userId}/${jobId}/${fileName}`; + + const info = getChunkDownloadInfo(filePath); + if (!info) { + set.status = 404; + return { error: "File not found" }; + } + + const chunkData = await getChunk(filePath, chunkIndex); + if (!chunkData) { + set.status = 404; + return { error: "Chunk not found" }; + } + + const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData); + + for (const [key, value] of Object.entries(headers)) { + set.headers[key] = value; + } + + return new Response(chunkData); + }, + { auth: true } + ) + /** + * Archive chunk 下載資訊 + */ + .get( + "/archive/:jobId/info", + async ({ params, user }) => { + const userId = user.id; + const job = await db + .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") + .get(user.id, params.jobId); + + if (!job) { + return { error: "Job not found" }; + } + + const jobId = decodeURIComponent(params.jobId); + const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`; + + if (!existsSync(archivePath)) { + return { error: "Archive not found" }; + } + + const info = getChunkDownloadInfo(archivePath); + + if (!info) { + return { error: "Archive not found" }; + } + + return { + ...info, + use_chunked: shouldUseChunkedDownload(archivePath), + }; + }, + { auth: true } + ) + /** + * Archive chunk 下載 + */ + .get( + "/archive/:jobId/chunk/:chunkIndex", + async ({ params, set, user }) => { + const userId = user.id; + const job = await db + .query("SELECT * FROM jobs WHERE user_id = ? AND id = ?") + .get(user.id, params.jobId); + + if (!job) { + set.status = 404; + return { error: "Job not found" }; + } + + const jobId = decodeURIComponent(params.jobId); + const chunkIndex = parseInt(params.chunkIndex, 10); + const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`; + + const info = getChunkDownloadInfo(archivePath); + if (!info) { + set.status = 404; + return { error: "Archive not found" }; + } + + const chunkData = await getChunk(archivePath, chunkIndex); + if (!chunkData) { + set.status = 404; + return { error: "Chunk not found" }; + } + + const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData); + + for (const [key, value] of Object.entries(headers)) { + set.headers[key] = value; + } + + return new Response(chunkData); + }, + { auth: true } + ); diff --git a/src/pages/uploadChunk.tsx b/src/pages/uploadChunk.tsx new file mode 100644 index 0000000..ad0dc3f --- /dev/null +++ b/src/pages/uploadChunk.tsx @@ -0,0 +1,88 @@ +/** + * Contents.CN Chunk 上傳 API + * + * 處理大檔的分段上傳 + */ + +import { Elysia, t } from "elysia"; +import { WEBROOT } from "../helpers/env"; +import { uploadsDir } from "../index"; +import { userService } from "./user"; +import sanitize from "sanitize-filename"; +import { + handleChunkUpload, + shouldUseChunkedUpload, + CHUNK_SIZE_BYTES, + calculateChunkCount, +} from "../transfer"; + +export const uploadChunk = new Elysia().use(userService).post( + "/upload-chunk", + async ({ body, user, cookie: { jobId } }) => { + if (!jobId?.value) { + return { + success: false, + message: "No active job session", + }; + } + + const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body; + + const sanitizedFileName = sanitize(file_name); + const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`; + + // 取得 chunk 資料 + const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk; + + const result = await handleChunkUpload( + upload_id, + parseInt(chunk_index, 10), + parseInt(total_chunks, 10), + chunkData, + sanitizedFileName, + parseInt(total_size, 10), + user.id, + jobId.value, + `${uploadsDir}${user.id}/`, + userUploadsDir + ); + + return result; + }, + { + body: t.Object({ + upload_id: t.String(), + chunk_index: t.String(), + total_chunks: t.String(), + file_name: t.String(), + total_size: t.String(), + chunk: t.File(), + }), + auth: true, + } +); + +/** + * 取得上傳資訊（用於前端判斷是否需要 chunk 上傳） + */ +export const uploadInfo = new Elysia().use(userService).post( + "/upload-info", + async ({ body }) => { + const { file_size } = body; + const size = parseInt(file_size, 10); + + const useChunked = shouldUseChunkedUpload(size); + + return { + use_chunked: useChunked, + chunk_size: CHUNK_SIZE_BYTES, + total_chunks: useChunked ? calculateChunkCount(size) : 1, + }; + }, + { + body: t.Object({ + file_size: t.String(), + }), + auth: true, + } +); diff --git a/src/transfer/archiveManager.ts b/src/transfer/archiveManager.ts new file mode 100644 index 0000000..5f76070 --- /dev/null +++ b/src/transfer/archiveManager.ts @@ -0,0 +1,154 @@ +/** + * Contents.CN 封裝管理器 + * + * 統一處理多檔輸出的封裝： + * - 唯一允許格式：.tar + * - 禁止：.tar.gz, .tgz, .zip 等 + */ + +import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs"; +import { join, basename } from "node:path"; +import * as tar from "tar"; +import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants"; + +/** + * 驗證封裝格式是否合法 + */ +export function validateArchiveFormat(fileName: string): boolean { + const lowerName = fileName.toLowerCase(); + + // 檢查是否使用禁止的格式 + for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) { + if (lowerName.endsWith(forbidden)) { + return false; + } + } + + // 檢查是否是允許的格式 + return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT); +} + +/** + * 取得正確的封裝檔名（強制使用 .tar） + */ +export function getArchiveFileName(baseName: string): string { + const lowerName = baseName.toLowerCase(); + + // 移除任何禁止的副檔名 + let cleanName = baseName; + for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) { + if (lowerName.endsWith(forbidden)) { + cleanName = baseName.slice(0, -forbidden.length); + break; + } + } + + // 如果已經是 .tar 結尾，直接返回 + if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) { + return cleanName; + } + + // 否則加上 .tar + return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`; +} + +/** + * 建立 .tar 封裝（不壓縮） + * + * @param sourceDir - 來源目錄 + * @param outputPath - 輸出的 .tar 檔案路徑 + * @param options - 額外選項 + */ +export async function createTarArchive( + sourceDir: string, + outputPath: string, + options: { + filter?: (path: string) => boolean; + prefix?: string; + } = {} +): Promise { + // 強制確保輸出是 .tar 格式 + const finalOutputPath = getArchiveFileName(outputPath); + + // 確保輸出目錄存在 + const outputDir = join(finalOutputPath, ".."); + if (!existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + + // 取得要封裝的檔案列表 + const files = readdirSync(sourceDir); + + // 預設過濾器：排除其他 tar 檔案 + const defaultFilter = (path: string) => !path.match(/\.tar$/i); + const filter = options.filter || defaultFilter; + + await tar.create( + { + file: finalOutputPath, + cwd: sourceDir, + filter: filter, + // 不使用 gzip 壓縮 + gzip: false, + }, + files.filter(f => filter(f)) + ); + + console.log(`Created tar archive: ${finalOutputPath}`); + return finalOutputPath; +} + +/** + * 建立用於多檔輸出的 .tar 封裝 + * + * @param outputDir - 輸出目錄（包含多個轉換結果） + * @param jobId - 任務 ID + */ +export async function createJobArchive(outputDir: string, jobId: string): Promise { + const archiveName = `converted_files_${jobId}.tar`; + const archivePath = join(outputDir, archiveName); + + await createTarArchive(outputDir, archivePath, { + filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案 + }); + + return archivePath; +} + +/** + * 從目錄封裝 MinerU 等多檔輸出引擎的結果 + * + * @param mineruOutputDir - MinerU 輸出目錄 + * @param targetPath - 目標 .tar 路徑 + */ +export async function createConverterArchive( + converterOutputDir: string, + targetPath: string +): Promise { + // 確保輸出使用 .tar 格式 + const tarPath = getArchiveFileName(targetPath); + + await createTarArchive(converterOutputDir, tarPath); + + return tarPath; +} + +/** + * 取得封裝檔案資訊 + */ +export function getArchiveInfo(archivePath: string): { + exists: boolean; + size: number; + fileName: string; +} | null { + if (!existsSync(archivePath)) { + return null; + } + + const stats = statSync(archivePath); + return { + exists: true, + size: stats.size, + fileName: basename(archivePath), + }; +} diff --git a/src/transfer/constants.ts b/src/transfer/constants.ts new file mode 100644 index 0000000..92b9c8c --- /dev/null +++ b/src/transfer/constants.ts @@ -0,0 +1,38 @@ +/** + * Contents.CN 全域檔案傳輸常數 + * + * 這些常數定義了整個系統的檔案傳輸規則 + */ + +/** + * 檔案大小門檻（10MB） + * - 小於等於此值：直接傳輸 + * - 大於此值：使用 chunk 分段傳輸 + */ +export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB + +/** + * 每個 chunk 的大小（5MB） + */ +export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB + +/** + * 上傳 session 過期時間（毫秒） + * 預設 1 小時 + */ +export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour + +/** + * 暫存目錄名稱 + */ +export const CHUNK_TEMP_DIR = "chunks_temp"; + +/** + * 允許的封裝格式（僅 .tar） + */ +export const ALLOWED_ARCHIVE_FORMAT = ".tar"; + +/** + * 禁止的封裝格式 + */ +export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"]; diff --git a/src/transfer/downloadManager.ts b/src/transfer/downloadManager.ts new file mode 100644 index 0000000..fede599 --- /dev/null +++ b/src/transfer/downloadManager.ts @@ -0,0 +1,123 @@ +/** + * Contents.CN 後端下載管理器 + * + * 統一處理所有檔案下載，包含： + * - 小檔（≤10MB）：直接回傳 + * - 大檔（>10MB）：chunk 分段下載 + */ + +import { existsSync, statSync, createReadStream } from "node:fs"; +import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants"; +import type { ChunkDownloadInfo } from "./types"; +import { getTransferMode } from "./types"; +import { basename } from "node:path"; + +/** + * 判斷檔案是否需要使用 chunk 下載 + */ +export function shouldUseChunkedDownload(filePath: string): boolean { + if (!existsSync(filePath)) { + return false; + } + const stats = statSync(filePath); + return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked"; +} + +/** + * 取得檔案的 chunk 下載資訊 + */ +export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null { + if (!existsSync(filePath)) { + return null; + } + + const stats = statSync(filePath); + const fileName = basename(filePath); + + return { + total_size: stats.size, + total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES), + chunk_size: CHUNK_SIZE_BYTES, + file_name: fileName, + }; +} + +/** + * 取得特定 chunk 的資料 + */ +export async function getChunk( + filePath: string, + chunkIndex: number, + chunkSize: number = CHUNK_SIZE_BYTES +): Promise { + if (!existsSync(filePath)) { + return null; + } + + const stats = statSync(filePath); + const start = chunkIndex * chunkSize; + const end = Math.min(start + chunkSize, stats.size); + + if (start >= stats.size) { + return null; + } + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + const stream = createReadStream(filePath, { start, end: end - 1 }); + + stream.on("data", (chunk) => { + chunks.push(chunk as Buffer); + }); + + stream.on("end", () => { + resolve(Buffer.concat(chunks)); + }); + + stream.on("error", reject); + }); +} + +/** + * 直接取得完整檔案（小檔用） + */ +export function getFileForDirectDownload(filePath: string): ReturnType | null { + if (!existsSync(filePath)) { + return null; + } + return Bun.file(filePath); +} + +/** + * 建立 chunk 下載回應的 headers + */ +export function createChunkDownloadHeaders( + info: ChunkDownloadInfo, + chunkIndex: number, + chunkData: Buffer +): Record { + const start = chunkIndex * info.chunk_size; + const end = start + chunkData.length - 1; + + return { + "Content-Type": "application/octet-stream", + "Content-Length": chunkData.length.toString(), + "Content-Range": `bytes ${start}-${end}/${info.total_size}`, + "Accept-Ranges": "bytes", + "X-Chunk-Index": chunkIndex.toString(), + "X-Total-Chunks": info.total_chunks.toString(), + "X-File-Name": encodeURIComponent(info.file_name), + "X-Total-Size": info.total_size.toString(), + }; +} + +/** + * 建立直接下載回應的 headers + */ +export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record { + return { + "Content-Type": "application/octet-stream", + "Content-Length": fileSize.toString(), + "Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`, + }; +} diff --git a/src/transfer/index.ts b/src/transfer/index.ts new file mode 100644 index 0000000..e15fdb2 --- /dev/null +++ b/src/transfer/index.ts @@ -0,0 +1,56 @@ +/** + * Contents.CN 全域檔案傳輸模組 + * + * 統一匯出所有傳輸相關功能 + */ + +// 常數 +export { + CHUNK_THRESHOLD_BYTES, + CHUNK_SIZE_BYTES, + UPLOAD_SESSION_TIMEOUT_MS, + CHUNK_TEMP_DIR, + ALLOWED_ARCHIVE_FORMAT, + FORBIDDEN_ARCHIVE_FORMATS, +} from "./constants"; + +// 類型 +export type { + ChunkUploadRequest, + ChunkUploadResponse, + DirectUploadResponse, + ChunkDownloadInfo, + UploadSession, + TransferMode, +} from "./types"; + +export { getTransferMode } from "./types"; + +// 上傳管理 +export { + uploadSessionManager, + shouldUseChunkedUpload, + handleDirectUpload, + handleChunkUpload, + calculateChunkCount, +} from "./uploadManager"; + +// 下載管理 +export { + shouldUseChunkedDownload, + getChunkDownloadInfo, + getChunk, + getFileForDirectDownload, + createChunkDownloadHeaders, + createDirectDownloadHeaders, +} from "./downloadManager"; + +// 封裝管理 +export { + validateArchiveFormat, + getArchiveFileName, + createTarArchive, + createJobArchive, + createConverterArchive, + getArchiveInfo, +} from "./archiveManager"; diff --git a/src/transfer/types.ts b/src/transfer/types.ts new file mode 100644 index 0000000..3d299fd --- /dev/null +++ b/src/transfer/types.ts @@ -0,0 +1,94 @@ +/** + * Contents.CN 檔案傳輸類型定義 + */ + +/** + * Chunk 上傳請求結構 + */ +export interface ChunkUploadRequest { + /** 上傳會話 ID（UUID） */ + upload_id: string; + /** 當前 chunk 索引（從 0 開始） */ + chunk_index: number; + /** 總 chunk 數量 */ + total_chunks: number; + /** chunk 數據 */ + data: Blob | ArrayBuffer; + /** 原始檔案名稱 */ + file_name: string; + /** 檔案總大小 */ + total_size: number; +} + +/** + * Chunk 上傳回應結構 + */ +export interface ChunkUploadResponse { + success: boolean; + message: string; + /** 已接收的 chunks 索引列表 */ + received_chunks?: number[]; + /** 是否所有 chunks 都已接收完成 */ + completed?: boolean; + /** 合併後的檔案路徑（僅完成時返回） */ + file_path?: string; +} + +/** + * 直接上傳回應結構 + */ +export interface DirectUploadResponse { + success: boolean; + message: string; + file_path?: string; +} + +/** + * Chunk 下載資訊 + */ +export interface ChunkDownloadInfo { + /** 檔案總大小 */ + total_size: number; + /** chunk 數量 */ + total_chunks: number; + /** 每個 chunk 大小 */ + chunk_size: number; + /** 檔案名稱 */ + file_name: string; +} + +/** + * 上傳會話狀態 + */ +export interface UploadSession { + /** 上傳 ID */ + upload_id: string; + /** 使用者 ID */ + user_id: string; + /** Job ID */ + job_id: string; + /** 檔案名稱 */ + file_name: string; + /** 檔案總大小 */ + total_size: number; + /** 總 chunk 數 */ + total_chunks: number; + /** 已接收的 chunks */ + received_chunks: Set; + /** 建立時間 */ + created_at: Date; + /** 暫存目錄路徑 */ + temp_dir: string; +} + +/** + * 傳輸模式 + */ +export type TransferMode = "direct" | "chunked"; + +/** + * 根據檔案大小決定傳輸模式 + */ +export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode { + return fileSize <= thresholdBytes ? "direct" : "chunked"; +} diff --git a/src/transfer/uploadManager.ts b/src/transfer/uploadManager.ts new file mode 100644 index 0000000..e737b58 --- /dev/null +++ b/src/transfer/uploadManager.ts @@ -0,0 +1,303 @@ +/** + * Contents.CN 後端上傳管理器 + * + * 統一處理所有檔案上傳，包含： + * - 小檔（≤10MB）：直接接收 + * - 大檔（>10MB）：chunk 接收與合併 + */ + +import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs"; +import { join } from "node:path"; +import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants"; +import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types"; +import { getTransferMode } from "./types"; + +/** + * 上傳會話管理器 + * 用於追蹤進行中的 chunk 上傳 + */ +class UploadSessionManager { + private sessions: Map = new Map(); + private cleanupInterval: ReturnType | null = null; + + constructor() { + // 定期清理過期的 sessions + this.cleanupInterval = setInterval(() => { + this.cleanupExpiredSessions(); + }, 5 * 60 * 1000); // 每 5 分鐘清理一次 + } + + /** + * 建立新的上傳會話 + */ + createSession( + uploadId: string, + userId: string, + jobId: string, + fileName: string, + totalSize: number, + totalChunks: number, + baseTempDir: string + ): UploadSession { + const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId); + + if (!existsSync(tempDir)) { + mkdirSync(tempDir, { recursive: true }); + } + + const session: UploadSession = { + upload_id: uploadId, + user_id: userId, + job_id: jobId, + file_name: fileName, + total_size: totalSize, + total_chunks: totalChunks, + received_chunks: new Set(), + created_at: new Date(), + temp_dir: tempDir, + }; + + this.sessions.set(uploadId, session); + return session; + } + + /** + * 取得上傳會話 + */ + getSession(uploadId: string): UploadSession | undefined { + return this.sessions.get(uploadId); + } + + /** + * 更新已接收的 chunk + */ + markChunkReceived(uploadId: string, chunkIndex: number): boolean { + const session = this.sessions.get(uploadId); + if (!session) return false; + + session.received_chunks.add(chunkIndex); + return true; + } + + /** + * 檢查是否所有 chunks 都已接收 + */ + isComplete(uploadId: string): boolean { + const session = this.sessions.get(uploadId); + if (!session) return false; + + return session.received_chunks.size === session.total_chunks; + } + + /** + * 移除會話並清理暫存檔案 + */ + removeSession(uploadId: string): void { + const session = this.sessions.get(uploadId); + if (session && existsSync(session.temp_dir)) { + rmSync(session.temp_dir, { recursive: true, force: true }); + } + this.sessions.delete(uploadId); + } + + /** + * 清理過期的會話 + */ + private cleanupExpiredSessions(): void { + const now = Date.now(); + for (const [uploadId, session] of this.sessions) { + if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) { + console.log(`Cleaning up expired upload session: ${uploadId}`); + this.removeSession(uploadId); + } + } + } + + /** + * 停止清理計時器 + */ + destroy(): void { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + this.cleanupInterval = null; + } + } +} + +// 單例實例 +export const uploadSessionManager = new UploadSessionManager(); + +/** + * 判斷檔案是否需要使用 chunk 上傳 + */ +export function shouldUseChunkedUpload(fileSize: number): boolean { + return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked"; +} + +/** + * 處理直接上傳（小檔） + */ +export async function handleDirectUpload( + file: File, + targetDir: string, + fileName: string +): Promise { + try { + const targetPath = join(targetDir, fileName); + + if (!existsSync(targetDir)) { + mkdirSync(targetDir, { recursive: true }); + } + + await Bun.write(targetPath, file); + + return { + success: true, + message: "File uploaded successfully.", + file_path: targetPath, + }; + } catch (error) { + console.error("Direct upload failed:", error); + return { + success: false, + message: `Upload failed: ${error}`, + }; + } +} + +/** + * 處理 chunk 上傳 + */ +export async function handleChunkUpload( + uploadId: string, + chunkIndex: number, + totalChunks: number, + chunkData: ArrayBuffer | Blob | Buffer, + fileName: string, + totalSize: number, + userId: string, + jobId: string, + baseTempDir: string, + targetDir: string +): Promise { + try { + // 取得或建立會話 + let session = uploadSessionManager.getSession(uploadId); + + if (!session) { + session = uploadSessionManager.createSession( + uploadId, + userId, + jobId, + fileName, + totalSize, + totalChunks, + baseTempDir + ); + } + + // 驗證會話資訊一致性 + if (session.total_chunks !== totalChunks || session.file_name !== fileName) { + return { + success: false, + message: "Session mismatch: inconsistent upload parameters", + }; + } + + // 儲存 chunk + const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`); + // 處理不同類型的 chunk 資料 + let data: ArrayBuffer | Buffer; + if (chunkData instanceof Blob) { + data = await chunkData.arrayBuffer(); + } else if (Buffer.isBuffer(chunkData)) { + data = chunkData; + } else { + data = chunkData; + } + await Bun.write(chunkPath, data); + + // 標記已接收 + uploadSessionManager.markChunkReceived(uploadId, chunkIndex); + + // 檢查是否完成 + if (uploadSessionManager.isComplete(uploadId)) { + // 合併所有 chunks + const finalPath = await mergeChunks(session, targetDir); + + // 清理會話 + uploadSessionManager.removeSession(uploadId); + + return { + success: true, + message: "Upload completed and merged successfully.", + received_chunks: Array.from({ length: totalChunks }, (_, i) => i), + completed: true, + file_path: finalPath, + }; + } + + return { + success: true, + message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`, + received_chunks: Array.from(session.received_chunks), + completed: false, + }; + } catch (error) { + console.error("Chunk upload failed:", error); + return { + success: false, + message: `Chunk upload failed: ${error}`, + }; + } +} + +/** + * 合併所有 chunks 為完整檔案 + */ +async function mergeChunks(session: UploadSession, targetDir: string): Promise { + const targetPath = join(targetDir, session.file_name); + + if (!existsSync(targetDir)) { + mkdirSync(targetDir, { recursive: true }); + } + + // 讀取並排序所有 chunks + const chunkFiles = readdirSync(session.temp_dir) + .filter(f => f.startsWith("chunk_")) + .sort(); + + // 建立輸出串流 + const writeStream = createWriteStream(targetPath); + + // 依序寫入每個 chunk + for (const chunkFile of chunkFiles) { + const chunkPath = join(session.temp_dir, chunkFile); + const chunkData = await Bun.file(chunkPath).arrayBuffer(); + writeStream.write(Buffer.from(chunkData)); + } + + // 結束寫入 + await new Promise((resolve, reject) => { + writeStream.end((err: Error | null) => { + if (err) reject(err); + else resolve(); + }); + }); + + // 驗證檔案大小 + const finalStats = statSync(targetPath); + if (finalStats.size !== session.total_size) { + console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`); + } + + console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`); + return targetPath; +} + +/** + * 計算需要的 chunk 數量 + */ +export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number { + return Math.ceil(fileSize / chunkSize); +} diff --git a/tests/converters/mineru.test.ts b/tests/converters/mineru.test.ts index d2226f0..181d302 100644 --- a/tests/converters/mineru.test.ts +++ b/tests/converters/mineru.test.ts @@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => { writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |"); callback(null, "MinerU conversion complete", ""); } else if (cmd === "tar") { - // Simulate tar.gz creation + // Simulate .tar creation (no compression) callback(null, "Archive created", ""); } }; - const targetPath = join(testDir, "output.tar.gz"); + // 使用 .tar 格式（不是 .tar.gz） + const targetPath = join(testDir, "output.tar"); await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); expect(mineruArgs).toContain("--table-mode"); @@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => { writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)"); callback(null, "MinerU conversion complete", ""); } else if (cmd === "tar") { - // Simulate tar.gz creation + // Simulate .tar creation (no compression) callback(null, "Archive created", ""); } }; - const targetPath = join(testDir, "output.tar.gz"); + // 使用 .tar 格式（不是 .tar.gz） + const targetPath = join(testDir, "output.tar"); await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile); expect(capturedArgs).toContain("--table-mode"); @@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => { }); }); -describe("MinerU converter tar.gz output", () => { - const testDir = "./test-output-mineru-targz"; +describe("MinerU converter .tar output (no compression)", () => { + const testDir = "./test-output-mineru-tar"; beforeEach(() => { if (!existsSync(testDir)) { @@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => { } }); - test("should create tar.gz archive from output", async () => { + test("should create .tar archive from output (without gzip)", async () => { let tarCalled = false; let tarArgs: string[] = []; @@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => { } }; - const targetPath = join(testDir, "output.tar.gz"); + // 使用 .tar 格式 + const targetPath = join(testDir, "output.tar"); const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); expect(result).toBe("Done"); expect(tarCalled).toBe(true); - expect(tarArgs).toContain("-czf"); + // 驗證使用 -cf 而非 -czf（不壓縮） + expect(tarArgs).toContain("-cf"); + // 確保沒有使用 gzip 壓縮標誌 + expect(tarArgs).not.toContain("-czf"); }); test("should reject on mineru error", async () => { @@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => { } }; - const targetPath = join(testDir, "output.tar.gz"); + const targetPath = join(testDir, "output.tar"); expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile)) .rejects.toMatch(/mineru error/); }); - test("should use correct tar arguments for compression", async () => { + test("should use correct tar arguments without compression", async () => { let tarArgs: string[] = []; const mockExecFile: ExecFileFn = ( @@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => { } }; - const targetPath = join(testDir, "test_MINERU_md-t.tar.gz"); + // 使用 .tar 格式（不是 .tar.gz） + const targetPath = join(testDir, "test_MINERU_md-t.tar"); await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); - // Verify tar is called with correct compression flags - expect(tarArgs[0]).toBe("-czf"); - expect(tarArgs[1]).toContain(".tar.gz"); + // 驗證 tar 使用正確的無壓縮標誌 + expect(tarArgs[0]).toBe("-cf"); + expect(tarArgs[1]).toContain(".tar"); + expect(tarArgs[1]).not.toContain(".tar.gz"); expect(tarArgs[2]).toBe("-C"); expect(tarArgs[4]).toBe("."); }); + + test("should convert .tar.gz target path to .tar", async () => { + let tarOutputPath = ""; + + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "mineru") { + const outputDir = args[3]; + if (outputDir && !existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + const autoDir = join(outputDir, "auto"); + if (!existsSync(autoDir)) { + mkdirSync(autoDir, { recursive: true }); + } + writeFileSync(join(autoDir, "output.md"), "# Test"); + callback(null, "Done", ""); + } else if (cmd === "tar") { + tarOutputPath = args[1]; // 輸出路徑 + callback(null, "Archive created", ""); + } + }; + + // 即使傳入 .tar.gz，也應該被轉換為 .tar + const targetPath = join(testDir, "output.tar.gz"); + await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); + + // 驗證輸出是 .tar 而非 .tar.gz + expect(tarOutputPath).toContain(".tar"); + expect(tarOutputPath).not.toContain(".tar.gz"); + }); }); diff --git a/tests/transfer/chunk-download.test.ts b/tests/transfer/chunk-download.test.ts new file mode 100644 index 0000000..b7beba7 --- /dev/null +++ b/tests/transfer/chunk-download.test.ts @@ -0,0 +1,208 @@ +/** + * Contents.CN Chunk 下載整合測試 + * + * 測試大檔 chunk 下載的完整流程 + */ + +import { test, expect, describe, beforeEach, afterEach } from "bun:test"; +import { mkdirSync, existsSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { + shouldUseChunkedDownload, + getChunkDownloadInfo, + getChunk, + createChunkDownloadHeaders, +} from "../../src/transfer/downloadManager"; +import { CHUNK_SIZE_BYTES, CHUNK_THRESHOLD_BYTES } from "../../src/transfer/constants"; + +const testDir = "./test-output-chunk-download"; + +describe("Chunk 下載資訊測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("getChunkDownloadInfo 應返回正確的下載資訊", () => { + const testFile = join(testDir, "large-file.bin"); + const fileSize = 25 * 1024 * 1024; // 25MB + + // 建立一個假檔案（只寫入少量資料模擬） + const content = Buffer.alloc(1024, "X"); // 1KB + writeFileSync(testFile, content); + + const info = getChunkDownloadInfo(testFile); + + expect(info).not.toBeNull(); + expect(info!.file_name).toBe("large-file.bin"); + expect(info!.total_size).toBe(1024); + expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES); + expect(info!.total_chunks).toBe(1); // 1KB 只需要 1 chunk + }); + + test("不存在的檔案應返回 null", () => { + const info = getChunkDownloadInfo(join(testDir, "non-existent.bin")); + expect(info).toBeNull(); + }); +}); + +describe("Chunk 讀取測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("getChunk 應正確讀取指定的 chunk", async () => { + const testFile = join(testDir, "chunked-file.bin"); + + // 建立測試檔案：20 bytes，分成 4 個 5-byte chunks + const content = "AAAAABBBBBCCCCCDDDD"; // 19 bytes + writeFileSync(testFile, content); + + // 讀取第一個 chunk（5 bytes） + const chunk0 = await getChunk(testFile, 0, 5); + expect(chunk0!.toString()).toBe("AAAAA"); + + // 讀取第二個 chunk + const chunk1 = await getChunk(testFile, 1, 5); + expect(chunk1!.toString()).toBe("BBBBB"); + + // 讀取第三個 chunk + const chunk2 = await getChunk(testFile, 2, 5); + expect(chunk2!.toString()).toBe("CCCCC"); + + // 讀取最後一個 chunk（只有 4 bytes） + const chunk3 = await getChunk(testFile, 3, 5); + expect(chunk3!.toString()).toBe("DDDD"); + }); + + test("讀取超出範圍的 chunk 應返回 null", async () => { + const testFile = join(testDir, "small-file.txt"); + writeFileSync(testFile, "Hello"); + + const chunk = await getChunk(testFile, 10, 5); // 超出範圍 + expect(chunk).toBeNull(); + }); + + test("不存在的檔案應返回 null", async () => { + const chunk = await getChunk(join(testDir, "non-existent.bin"), 0, 5); + expect(chunk).toBeNull(); + }); +}); + +describe("Chunk 下載 Headers 測試", () => { + test("createChunkDownloadHeaders 應生成正確的 headers", () => { + const info = { + total_size: 1000, + total_chunks: 4, + chunk_size: 256, + file_name: "test-file.bin", + }; + + const chunkData = Buffer.alloc(256, "X"); + const headers = createChunkDownloadHeaders(info, 0, chunkData); + + expect(headers["Content-Type"]).toBe("application/octet-stream"); + expect(headers["Content-Length"]).toBe("256"); + expect(headers["Content-Range"]).toBe("bytes 0-255/1000"); + expect(headers["Accept-Ranges"]).toBe("bytes"); + expect(headers["X-Chunk-Index"]).toBe("0"); + expect(headers["X-Total-Chunks"]).toBe("4"); + expect(headers["X-Total-Size"]).toBe("1000"); + }); + + test("最後一個 chunk 的 headers 應正確", () => { + const info = { + total_size: 1000, + total_chunks: 4, + chunk_size: 256, + file_name: "test-file.bin", + }; + + // 最後一個 chunk 只有 232 bytes (1000 - 256*3) + const chunkData = Buffer.alloc(232, "Z"); + const headers = createChunkDownloadHeaders(info, 3, chunkData); + + expect(headers["Content-Length"]).toBe("232"); + expect(headers["Content-Range"]).toBe("bytes 768-999/1000"); + expect(headers["X-Chunk-Index"]).toBe("3"); + }); +}); + +describe("傳輸模式判斷測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("小檔（≤10MB）不應使用 chunk 下載", () => { + const smallFile = join(testDir, "small.txt"); + writeFileSync(smallFile, "Small file content"); + + expect(shouldUseChunkedDownload(smallFile)).toBe(false); + }); + + test("不存在的檔案不應使用 chunk 下載", () => { + expect(shouldUseChunkedDownload(join(testDir, "non-existent.bin"))).toBe(false); + }); +}); + +describe("完整 chunk 下載流程測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("應能完整讀取並合併所有 chunks", async () => { + const testFile = join(testDir, "complete-file.bin"); + const originalContent = "AAAAAABBBBBBCCCCCCDDDDDDEEEEEE"; // 30 bytes + writeFileSync(testFile, originalContent); + + const info = getChunkDownloadInfo(testFile); + expect(info).not.toBeNull(); + + // 模擬分段讀取（每個 chunk 10 bytes） + const chunkSize = 10; + const totalChunks = Math.ceil(originalContent.length / chunkSize); + const chunks: Buffer[] = []; + + for (let i = 0; i < totalChunks; i++) { + const chunk = await getChunk(testFile, i, chunkSize); + if (chunk) { + chunks.push(chunk); + } + } + + // 合併所有 chunks + const mergedContent = Buffer.concat(chunks).toString(); + expect(mergedContent).toBe(originalContent); + }); +}); diff --git a/tests/transfer/chunk-upload.test.ts b/tests/transfer/chunk-upload.test.ts new file mode 100644 index 0000000..4dc05b6 --- /dev/null +++ b/tests/transfer/chunk-upload.test.ts @@ -0,0 +1,178 @@ +/** + * Contents.CN Chunk 上傳整合測試 + * + * 測試大檔 chunk 上傳的完整流程 + */ + +import { test, expect, describe, beforeEach, afterEach } from "bun:test"; +import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { + handleChunkUpload, + uploadSessionManager, + calculateChunkCount, +} from "../../src/transfer/uploadManager"; +import { CHUNK_SIZE_BYTES } from "../../src/transfer/constants"; + +const testDir = "./test-output-chunk-upload"; + +describe("Chunk 上傳整合測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("多 chunk 上傳應正確合併", async () => { + const uploadId = "test-chunked-upload-1"; + const fileName = "large-file.bin"; + const userId = "test-user"; + const jobId = "test-job"; + + // 建立測試資料（模擬 15MB 檔案，分 3 個 chunks） + const chunkSize = 5 * 1024; // 使用較小的 chunk 以便測試 + const chunk1 = Buffer.alloc(chunkSize, "A"); + const chunk2 = Buffer.alloc(chunkSize, "B"); + const chunk3 = Buffer.alloc(chunkSize, "C"); + const totalSize = chunkSize * 3; + const totalChunks = 3; + + const baseTempDir = join(testDir, "temp"); + const targetDir = join(testDir, "uploads"); + mkdirSync(baseTempDir, { recursive: true }); + mkdirSync(targetDir, { recursive: true }); + + // 上傳 chunk 1 + const result1 = await handleChunkUpload( + uploadId, 0, totalChunks, chunk1, fileName, totalSize, + userId, jobId, baseTempDir, targetDir + ); + expect(result1.success).toBe(true); + expect(result1.completed).toBe(false); + expect(result1.received_chunks).toContain(0); + + // 上傳 chunk 2 + const result2 = await handleChunkUpload( + uploadId, 1, totalChunks, chunk2, fileName, totalSize, + userId, jobId, baseTempDir, targetDir + ); + expect(result2.success).toBe(true); + expect(result2.completed).toBe(false); + + // 上傳 chunk 3（最後一個） + const result3 = await handleChunkUpload( + uploadId, 2, totalChunks, chunk3, fileName, totalSize, + userId, jobId, baseTempDir, targetDir + ); + expect(result3.success).toBe(true); + expect(result3.completed).toBe(true); + expect(result3.file_path).toBeDefined(); + + // 驗證合併後的檔案 + const mergedFile = join(targetDir, fileName); + expect(existsSync(mergedFile)).toBe(true); + + const mergedContent = readFileSync(mergedFile); + expect(mergedContent.length).toBe(totalSize); + + // 驗證內容正確性 + const expectedContent = Buffer.concat([chunk1, chunk2, chunk3]); + expect(mergedContent.equals(expectedContent)).toBe(true); + }); + + test("亂序上傳 chunks 也應正確合併", async () => { + const uploadId = "test-chunked-upload-2"; + const fileName = "out-of-order.bin"; + const userId = "test-user"; + const jobId = "test-job"; + + const chunkSize = 1024; + const chunk0 = Buffer.alloc(chunkSize, "0"); + const chunk1 = Buffer.alloc(chunkSize, "1"); + const chunk2 = Buffer.alloc(chunkSize, "2"); + const chunk3 = Buffer.alloc(chunkSize, "3"); + const totalSize = chunkSize * 4; + const totalChunks = 4; + + const baseTempDir = join(testDir, "temp2"); + const targetDir = join(testDir, "uploads2"); + mkdirSync(baseTempDir, { recursive: true }); + mkdirSync(targetDir, { recursive: true }); + + // 亂序上傳：3, 1, 0, 2 + await handleChunkUpload(uploadId, 3, totalChunks, chunk3, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + const finalResult = await handleChunkUpload(uploadId, 2, totalChunks, chunk2, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + + expect(finalResult.success).toBe(true); + expect(finalResult.completed).toBe(true); + + // 驗證檔案順序正確 + const mergedFile = join(targetDir, fileName); + const mergedContent = readFileSync(mergedFile); + + // 即使亂序上傳，合併後應該按正確順序 + const expectedContent = Buffer.concat([chunk0, chunk1, chunk2, chunk3]); + expect(mergedContent.equals(expectedContent)).toBe(true); + }); + + test("重複上傳相同 chunk 應被處理", async () => { + const uploadId = "test-chunked-upload-3"; + const fileName = "duplicate-chunk.bin"; + const userId = "test-user"; + const jobId = "test-job"; + + const chunkSize = 512; + const chunk0 = Buffer.alloc(chunkSize, "X"); + const chunk1 = Buffer.alloc(chunkSize, "Y"); + const totalSize = chunkSize * 2; + const totalChunks = 2; + + const baseTempDir = join(testDir, "temp3"); + const targetDir = join(testDir, "uploads3"); + mkdirSync(baseTempDir, { recursive: true }); + mkdirSync(targetDir, { recursive: true }); + + // 上傳 chunk 0 + await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + + // 重複上傳 chunk 0 + await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + + // 上傳 chunk 1 + const result = await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir); + + expect(result.success).toBe(true); + expect(result.completed).toBe(true); + + // 驗證檔案大小正確（不應該因為重複上傳而變大） + const mergedFile = join(targetDir, fileName); + const stats = statSync(mergedFile); + expect(stats.size).toBe(totalSize); + }); +}); + +describe("Chunk 數量計算測試", () => { + test("應正確計算不同大小檔案的 chunk 數量", () => { + const chunkSize = CHUNK_SIZE_BYTES; + + // 剛好 1 個 chunk + expect(calculateChunkCount(chunkSize)).toBe(1); + + // 多 1 byte 需要 2 個 chunks + expect(calculateChunkCount(chunkSize + 1)).toBe(2); + + // 50MB 需要 10 個 chunks + expect(calculateChunkCount(50 * 1024 * 1024)).toBe(10); + + // 1 byte 需要 1 個 chunk + expect(calculateChunkCount(1)).toBe(1); + }); +}); diff --git a/tests/transfer/transfer.test.ts b/tests/transfer/transfer.test.ts new file mode 100644 index 0000000..761c148 --- /dev/null +++ b/tests/transfer/transfer.test.ts @@ -0,0 +1,331 @@ +/** + * Contents.CN 全域檔案傳輸機制測試 + * + * 測試項目： + * 1. 小檔直傳測試（≤10MB） + * 2. 大檔 chunk 上傳測試（>10MB） + * 3. chunk 合併正確性測試 + * 4. .tar 封裝測試 + * 5. End-to-End 測試 + */ + +import { test, expect, describe, beforeEach, afterEach } from "bun:test"; +import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { + shouldUseChunkedUpload, + calculateChunkCount, + handleDirectUpload, + handleChunkUpload, + uploadSessionManager, +} from "../../src/transfer/uploadManager"; +import { + shouldUseChunkedDownload, + getChunkDownloadInfo, + getChunk, +} from "../../src/transfer/downloadManager"; +import { + validateArchiveFormat, + getArchiveFileName, + createTarArchive, +} from "../../src/transfer/archiveManager"; +import { + CHUNK_THRESHOLD_BYTES, + CHUNK_SIZE_BYTES, + ALLOWED_ARCHIVE_FORMAT, + FORBIDDEN_ARCHIVE_FORMATS, +} from "../../src/transfer/constants"; +import { getTransferMode } from "../../src/transfer/types"; + +const testDir = "./test-output-transfer"; + +describe("傳輸常數測試", () => { + test("CHUNK_THRESHOLD_BYTES 應為 10MB", () => { + expect(CHUNK_THRESHOLD_BYTES).toBe(10 * 1024 * 1024); + }); + + test("CHUNK_SIZE_BYTES 應為 5MB", () => { + expect(CHUNK_SIZE_BYTES).toBe(5 * 1024 * 1024); + }); + + test("唯一允許的封裝格式應為 .tar", () => { + expect(ALLOWED_ARCHIVE_FORMAT).toBe(".tar"); + }); + + test("禁止的封裝格式應包含 .tar.gz, .tgz, .zip, .gz", () => { + expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tar.gz"); + expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tgz"); + expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".zip"); + expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".gz"); + }); +}); + +describe("傳輸模式判斷測試", () => { + test("≤10MB 應使用直傳模式", () => { + expect(getTransferMode(10 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct"); + expect(getTransferMode(5 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct"); + expect(getTransferMode(1024, CHUNK_THRESHOLD_BYTES)).toBe("direct"); + expect(getTransferMode(0, CHUNK_THRESHOLD_BYTES)).toBe("direct"); + }); + + test(">10MB 應使用 chunk 模式", () => { + expect(getTransferMode(10 * 1024 * 1024 + 1, CHUNK_THRESHOLD_BYTES)).toBe("chunked"); + expect(getTransferMode(50 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked"); + expect(getTransferMode(100 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked"); + }); + + test("shouldUseChunkedUpload 函數應正確判斷", () => { + expect(shouldUseChunkedUpload(10 * 1024 * 1024)).toBe(false); + expect(shouldUseChunkedUpload(10 * 1024 * 1024 + 1)).toBe(true); + }); +}); + +describe("Chunk 數量計算測試", () => { + test("計算 chunk 數量應正確", () => { + // 10MB / 5MB = 2 chunks + expect(calculateChunkCount(10 * 1024 * 1024)).toBe(2); + + // 15MB / 5MB = 3 chunks + expect(calculateChunkCount(15 * 1024 * 1024)).toBe(3); + + // 1 byte 也需要 1 chunk + expect(calculateChunkCount(1)).toBe(1); + + // 5MB + 1 byte = 2 chunks + expect(calculateChunkCount(5 * 1024 * 1024 + 1)).toBe(2); + }); +}); + +describe("封裝格式驗證測試", () => { + test("validateArchiveFormat 應正確驗證格式", () => { + // 允許的格式 + expect(validateArchiveFormat("output.tar")).toBe(true); + expect(validateArchiveFormat("my_archive.tar")).toBe(true); + + // 禁止的格式 + expect(validateArchiveFormat("output.tar.gz")).toBe(false); + expect(validateArchiveFormat("output.tgz")).toBe(false); + expect(validateArchiveFormat("output.zip")).toBe(false); + expect(validateArchiveFormat("output.gz")).toBe(false); + }); + + test("getArchiveFileName 應強制轉換為 .tar", () => { + // 已經是 .tar + expect(getArchiveFileName("output.tar")).toBe("output.tar"); + + // 轉換 .tar.gz -> .tar + expect(getArchiveFileName("output.tar.gz")).toBe("output.tar"); + + // 轉換 .tgz -> .tar + expect(getArchiveFileName("output.tgz")).toBe("output.tar"); + + // 無副檔名 -> 加上 .tar + expect(getArchiveFileName("output")).toBe("output.tar"); + + // .zip 也是禁止的格式，會被移除並加上 .tar + expect(getArchiveFileName("output.zip")).toBe("output.tar"); + + // .gz 也是禁止的格式 + expect(getArchiveFileName("output.gz")).toBe("output.tar"); + }); +}); + +describe("上傳管理測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("小檔直傳應成功", async () => { + const targetDir = join(testDir, "uploads"); + const testContent = "Hello, World!"; + const file = new File([testContent], "test.txt", { type: "text/plain" }); + + const result = await handleDirectUpload(file as any, targetDir, "test.txt"); + + expect(result.success).toBe(true); + expect(existsSync(join(targetDir, "test.txt"))).toBe(true); + expect(readFileSync(join(targetDir, "test.txt"), "utf-8")).toBe(testContent); + }); +}); + +describe("下載管理測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("小檔不應使用 chunk 下載", () => { + const smallFile = join(testDir, "small.txt"); + writeFileSync(smallFile, "Small content"); + + expect(shouldUseChunkedDownload(smallFile)).toBe(false); + }); + + test("getChunkDownloadInfo 應返回正確資訊", () => { + const testFile = join(testDir, "test.txt"); + const content = "Test content for download"; + writeFileSync(testFile, content); + + const info = getChunkDownloadInfo(testFile); + + expect(info).not.toBeNull(); + expect(info!.file_name).toBe("test.txt"); + expect(info!.total_size).toBe(content.length); + expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES); + }); + + test("getChunk 應正確讀取 chunk 資料", async () => { + const testFile = join(testDir, "test.txt"); + const content = "ABCDEFGHIJ"; // 10 bytes + writeFileSync(testFile, content); + + // 讀取第一個 chunk（整個檔案，因為小於 chunk size） + const chunk = await getChunk(testFile, 0, 5); + + expect(chunk).not.toBeNull(); + expect(chunk!.toString()).toBe("ABCDE"); + + // 讀取第二個 chunk + const chunk2 = await getChunk(testFile, 1, 5); + expect(chunk2!.toString()).toBe("FGHIJ"); + }); +}); + +describe(".tar 封裝測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("createTarArchive 應建立 .tar 檔案", async () => { + const sourceDir = join(testDir, "source"); + const outputTar = join(testDir, "output.tar"); + + mkdirSync(sourceDir, { recursive: true }); + writeFileSync(join(sourceDir, "file1.txt"), "Content 1"); + writeFileSync(join(sourceDir, "file2.txt"), "Content 2"); + + const result = await createTarArchive(sourceDir, outputTar); + + expect(existsSync(result)).toBe(true); + expect(result.endsWith(".tar")).toBe(true); + expect(result).not.toContain(".tar.gz"); + }); + + test("createTarArchive 應強制使用 .tar 格式", async () => { + const sourceDir = join(testDir, "source2"); + // 即使傳入 .tar.gz，也應該輸出 .tar + const outputTar = join(testDir, "output.tar.gz"); + + mkdirSync(sourceDir, { recursive: true }); + writeFileSync(join(sourceDir, "file.txt"), "Content"); + + const result = await createTarArchive(sourceDir, outputTar); + + // 結果應該是 .tar 而非 .tar.gz + expect(result.endsWith(".tar")).toBe(true); + expect(result).not.toContain(".tar.gz"); + }); +}); + +describe("End-to-End 測試", () => { + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("完整流程：上傳 -> 封裝 -> 下載資訊", async () => { + // 1. 模擬上傳 + const uploadsDir = join(testDir, "uploads"); + const outputDir = join(testDir, "output"); + mkdirSync(uploadsDir, { recursive: true }); + mkdirSync(outputDir, { recursive: true }); + + const uploadedFile = join(uploadsDir, "document.pdf"); + writeFileSync(uploadedFile, "Fake PDF content"); + + // 2. 模擬轉換輸出 + const conversionOutput = join(outputDir, "conversion_result"); + mkdirSync(conversionOutput, { recursive: true }); + writeFileSync(join(conversionOutput, "output.md"), "# Converted Document"); + mkdirSync(join(conversionOutput, "images"), { recursive: true }); + writeFileSync(join(conversionOutput, "images", "fig1.png"), "Fake image"); + + // 3. 建立 .tar 封裝 + const tarPath = join(outputDir, "result.tar"); + const archivePath = await createTarArchive(conversionOutput, tarPath); + + expect(existsSync(archivePath)).toBe(true); + expect(archivePath.endsWith(".tar")).toBe(true); + + // 4. 取得下載資訊 + const downloadInfo = getChunkDownloadInfo(archivePath); + expect(downloadInfo).not.toBeNull(); + expect(downloadInfo!.file_name).toBe("result.tar"); + + // 5. 驗證傳輸模式判斷 + const fileStats = statSync(archivePath); + const shouldChunk = fileStats.size > CHUNK_THRESHOLD_BYTES; + expect(shouldUseChunkedDownload(archivePath)).toBe(shouldChunk); + }); +}); + +describe("上傳會話管理測試", () => { + test("應正確建立和管理上傳會話", () => { + const uploadId = "test-upload-123"; + const session = uploadSessionManager.createSession( + uploadId, + "user-1", + "job-1", + "large-file.pdf", + 50 * 1024 * 1024, // 50MB + 10, // 10 chunks + testDir + ); + + expect(session.upload_id).toBe(uploadId); + expect(session.total_chunks).toBe(10); + expect(session.received_chunks.size).toBe(0); + + // 標記已接收 chunks + uploadSessionManager.markChunkReceived(uploadId, 0); + uploadSessionManager.markChunkReceived(uploadId, 1); + + const updatedSession = uploadSessionManager.getSession(uploadId); + expect(updatedSession!.received_chunks.size).toBe(2); + expect(uploadSessionManager.isComplete(uploadId)).toBe(false); + + // 清理 + uploadSessionManager.removeSession(uploadId); + expect(uploadSessionManager.getSession(uploadId)).toBeUndefined(); + }); +});