feat: 實作全域檔案傳輸機制與 .tar 封裝規範

## 主要變更

### 新增全域檔案傳輸模組 (src/transfer/)
- constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小)
- types.ts: 傳輸類型定義
- uploadManager.ts: 後端上傳管理器(支援直傳與 chunk)
- downloadManager.ts: 後端下載管理器(支援直傳與 chunk)
- archiveManager.ts: 封裝管理器(僅允許 .tar)
- index.ts: 統一匯出

### 新增 API 端點
- uploadChunk.tsx: Chunk 上傳 API
- downloadChunk.tsx: Chunk 下載 API

### 前端更新
- public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk)
- public/transfer.js: 前端傳輸管理模組

### 轉換器更新
- mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz
- download.tsx: 使用統一的封裝管理器

### 測試
- tests/transfer/: 完整傳輸機制測試套件
- tests/converters/mineru.test.ts: 更新以符合 .tar 規範

### 文件
- README.md: 新增檔案傳輸機制說明

## 設計原則
- 檔案 ≤10MB:直接傳輸
- 檔案 >10MB:使用 5MB chunks 分段傳輸
- 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip)
- 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
Your Name 2026-01-21 13:58:01 +08:00
parent f7ebc084ea
commit 5322f85721
18 changed files with 2387 additions and 61 deletions

View file

@ -158,7 +158,7 @@ docker compose up -d
## 支援格式 ## 支援格式
| 轉換器 | 用途 | 格式數 | | 轉換器 | 用途 | 格式數 |
| ----------- | ------ | ------ | | ----------- | ------- | ------ |
| FFmpeg | 影音 | 400+ | | FFmpeg | 影音 | 400+ |
| ImageMagick | 圖片 | 200+ | | ImageMagick | 圖片 | 200+ |
| LibreOffice | 文件 | 60+ | | LibreOffice | 文件 | 60+ |
@ -175,10 +175,32 @@ docker compose up -d
ConvertX 內建文件轉換引擎。 ConvertX 內建文件轉換引擎。
- md-t - md-t(表格以 Markdown 呈現)
- md-i - md-i(表格以圖片呈現)
輸出格式tar.gz 輸出格式:.tar不壓縮封裝
---
## 檔案傳輸機制
Contents.CN 使用統一的檔案傳輸策略:
| 檔案大小 | 傳輸方式 | 說明 |
| -------- | -------- | ---- |
| ≤ 10MB | 直接傳輸 | 單一請求完成上傳/下載 |
| > 10MB | 分段傳輸 | 使用 5MB chunks 分段傳輸 |
### 多檔輸出封裝
- ✅ 唯一允許格式:`.tar`
- ❌ 禁止使用:`.tar.gz``.tgz``.zip`
### 設計原則
- 分段傳輸僅存在於傳輸層
- 轉換引擎只接收完整檔案
- 前後端使用共用傳輸模組
--- ---

View file

@ -242,11 +242,54 @@ const deleteRow = (target) => {
}).catch((err) => console.log(err)); }).catch((err) => console.log(err));
}; };
// ==================== 全域傳輸常數 ====================
const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
/**
* 判斷是否需要使用 chunk 傳輸
*/
function shouldUseChunkedUpload(fileSize) {
return fileSize > CHUNK_THRESHOLD_BYTES;
}
/**
* 生成 UUID
*/
function generateUploadId() {
if (typeof crypto !== "undefined" && crypto.randomUUID) {
return crypto.randomUUID();
}
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) {
const r = Math.random() * 16 | 0;
const v = c === "x" ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}
/**
* 統一上傳檔案自動判斷使用直傳或 chunk
*
* @param {File} file - 要上傳的檔案
*/
const uploadFile = (file) => { const uploadFile = (file) => {
convertButton.disabled = true; convertButton.disabled = true;
convertButton.value = getTranslation("convert", "uploading"); convertButton.value = getTranslation("convert", "uploading");
pendingFiles += 1; pendingFiles += 1;
if (shouldUseChunkedUpload(file.size)) {
// 大檔:使用 chunk 上傳
uploadFileChunked(file);
} else {
// 小檔:直接上傳
uploadFileDirect(file);
}
};
/**
* 直接上傳10MB
*/
const uploadFileDirect = (file) => {
const formData = new FormData(); const formData = new FormData();
formData.append("file", file, file.name); formData.append("file", file, file.name);
@ -255,7 +298,12 @@ const uploadFile = (file) => {
xhr.open("POST", `${webroot}/upload`, true); xhr.open("POST", `${webroot}/upload`, true);
xhr.onload = () => { xhr.onload = () => {
let data = JSON.parse(xhr.responseText); let data = {};
try {
data = JSON.parse(xhr.responseText);
} catch (e) {
console.log("Parse error:", e);
}
pendingFiles -= 1; pendingFiles -= 1;
if (pendingFiles === 0) { if (pendingFiles === 0) {
@ -265,10 +313,12 @@ const uploadFile = (file) => {
convertButton.value = getTranslation("convert", "convertButton"); convertButton.value = getTranslation("convert", "convertButton");
} }
//Remove the progress bar when upload is done // Remove the progress bar when upload is done
let progressbar = file.htmlRow.getElementsByTagName("progress"); let progressbar = file.htmlRow.getElementsByTagName("progress");
if (progressbar[0]) {
progressbar[0].parentElement.remove(); progressbar[0].parentElement.remove();
console.log(data); }
console.log("Direct upload complete:", data);
}; };
xhr.upload.onprogress = (e) => { xhr.upload.onprogress = (e) => {
@ -277,16 +327,88 @@ const uploadFile = (file) => {
console.log(`upload progress (${file.name}):`, (100 * sent) / total); console.log(`upload progress (${file.name}):`, (100 * sent) / total);
let progressbar = file.htmlRow.getElementsByTagName("progress"); let progressbar = file.htmlRow.getElementsByTagName("progress");
if (progressbar[0]) {
progressbar[0].value = (100 * sent) / total; progressbar[0].value = (100 * sent) / total;
}
}; };
xhr.onerror = (e) => { xhr.onerror = (e) => {
console.log(e); console.log("Upload error:", e);
pendingFiles -= 1;
if (pendingFiles === 0) {
convertButton.value = getTranslation("convert", "convertButton");
}
}; };
xhr.send(formData); xhr.send(formData);
}; };
/**
* Chunk 上傳>10MB
*/
const uploadFileChunked = async (file) => {
const uploadId = generateUploadId();
const totalChunks = Math.ceil(file.size / CHUNK_SIZE_BYTES);
console.log(`Starting chunked upload: ${file.name}, size: ${file.size}, chunks: ${totalChunks}`);
try {
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
const start = chunkIndex * CHUNK_SIZE_BYTES;
const end = Math.min(start + CHUNK_SIZE_BYTES, file.size);
const chunk = file.slice(start, end);
const formData = new FormData();
formData.append("upload_id", uploadId);
formData.append("chunk_index", chunkIndex.toString());
formData.append("total_chunks", totalChunks.toString());
formData.append("file_name", file.name);
formData.append("total_size", file.size.toString());
formData.append("chunk", chunk);
const response = await fetch(`${webroot}/upload-chunk`, {
method: "POST",
body: formData,
});
if (!response.ok) {
throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`);
}
// 更新進度
const percent = ((chunkIndex + 1) / totalChunks) * 100;
let progressbar = file.htmlRow.getElementsByTagName("progress");
if (progressbar[0]) {
progressbar[0].value = percent;
}
console.log(`Chunk ${chunkIndex + 1}/${totalChunks} uploaded (${percent.toFixed(1)}%)`);
}
// 完成
pendingFiles -= 1;
if (pendingFiles === 0) {
if (formatSelected) {
convertButton.disabled = false;
}
convertButton.value = getTranslation("convert", "convertButton");
}
// Remove the progress bar
let progressbar = file.htmlRow.getElementsByTagName("progress");
if (progressbar[0]) {
progressbar[0].parentElement.remove();
}
console.log("Chunked upload complete:", file.name);
} catch (error) {
console.error("Chunked upload failed:", error);
pendingFiles -= 1;
if (pendingFiles === 0) {
convertButton.value = getTranslation("convert", "convertButton");
}
}
};
const formConvert = document.querySelector(`form[action='${webroot}/convert']`); const formConvert = document.querySelector(`form[action='${webroot}/convert']`);
formConvert.addEventListener("submit", () => { formConvert.addEventListener("submit", () => {

388
public/transfer.js Normal file
View file

@ -0,0 +1,388 @@
/**
* Contents.CN 前端檔案傳輸管理器
*
* 統一處理所有檔案的上傳與下載
* - 檔案 10MB直接傳輸
* - 檔案 > 10MB使用 chunk 分段傳輸
*
* 重要所有功能必須使用此模組不得自行實作傳輸邏輯
*/
// ==================== 常數定義 ====================
/**
* 檔案大小門檻10MB
*/
const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024;
/**
* 每個 chunk 的大小5MB
*/
const CHUNK_SIZE_BYTES = 5 * 1024 * 1024;
// ==================== 工具函數 ====================
/**
* 生成 UUID用於 upload_id
*/
function generateUploadId() {
if (typeof crypto !== "undefined" && crypto.randomUUID) {
return crypto.randomUUID();
}
// 降級方案
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(/[xy]/g, function(c) {
const r = Math.random() * 16 | 0;
const v = c === "x" ? r : (r & 0x3 | 0x8);
return v.toString(16);
});
}
/**
* 判斷是否需要使用 chunk 傳輸
*/
function shouldUseChunkedTransfer(fileSize) {
return fileSize > CHUNK_THRESHOLD_BYTES;
}
/**
* 計算 chunk 數量
*/
function calculateChunkCount(fileSize) {
return Math.ceil(fileSize / CHUNK_SIZE_BYTES);
}
// ==================== 上傳管理器 ====================
/**
* 上傳管理器類別
*/
class UploadManager {
/**
* @param {string} webroot - 網站根路徑
*/
constructor(webroot) {
this.webroot = webroot;
this.activeUploads = new Map();
}
/**
* 上傳檔案自動判斷使用直傳或 chunk
*
* @param {File} file - 要上傳的檔案
* @param {object} options - 選項
* @param {function} options.onProgress - 進度回調 (percent: number) => void
* @param {function} options.onComplete - 完成回調 (response: object) => void
* @param {function} options.onError - 錯誤回調 (error: Error) => void
* @returns {Promise<object>} 上傳結果
*/
async uploadFile(file, options = {}) {
const { onProgress, onComplete, onError } = options;
try {
let result;
if (shouldUseChunkedTransfer(file.size)) {
// 大檔:使用 chunk 上傳
result = await this.uploadChunked(file, onProgress);
} else {
// 小檔:直接上傳
result = await this.uploadDirect(file, onProgress);
}
if (onComplete) onComplete(result);
return result;
} catch (error) {
if (onError) onError(error);
throw error;
}
}
/**
* 直接上傳小檔
*/
async uploadDirect(file, onProgress) {
return new Promise((resolve, reject) => {
const formData = new FormData();
formData.append("file", file, file.name);
const xhr = new XMLHttpRequest();
xhr.open("POST", `${this.webroot}/upload`, true);
xhr.upload.onprogress = (e) => {
if (e.lengthComputable && onProgress) {
const percent = (e.loaded / e.total) * 100;
onProgress(percent);
}
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) {
try {
const data = JSON.parse(xhr.responseText);
resolve(data);
} catch {
resolve({ success: true, message: "Upload completed" });
}
} else {
reject(new Error(`Upload failed with status ${xhr.status}`));
}
};
xhr.onerror = () => reject(new Error("Upload failed"));
xhr.send(formData);
});
}
/**
* Chunk 上傳大檔
*/
async uploadChunked(file, onProgress) {
const uploadId = generateUploadId();
const totalChunks = calculateChunkCount(file.size);
this.activeUploads.set(uploadId, {
file,
totalChunks,
uploadedChunks: 0,
status: "uploading"
});
try {
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
const start = chunkIndex * CHUNK_SIZE_BYTES;
const end = Math.min(start + CHUNK_SIZE_BYTES, file.size);
const chunk = file.slice(start, end);
await this.uploadChunk(uploadId, chunkIndex, totalChunks, chunk, file.name, file.size);
// 更新進度
const uploadInfo = this.activeUploads.get(uploadId);
if (uploadInfo) {
uploadInfo.uploadedChunks = chunkIndex + 1;
const percent = ((chunkIndex + 1) / totalChunks) * 100;
if (onProgress) onProgress(percent);
}
}
this.activeUploads.delete(uploadId);
return { success: true, message: "Chunked upload completed", upload_id: uploadId };
} catch (error) {
this.activeUploads.delete(uploadId);
throw error;
}
}
/**
* 上傳單個 chunk
*/
async uploadChunk(uploadId, chunkIndex, totalChunks, chunkData, fileName, totalSize) {
const formData = new FormData();
formData.append("upload_id", uploadId);
formData.append("chunk_index", chunkIndex.toString());
formData.append("total_chunks", totalChunks.toString());
formData.append("file_name", fileName);
formData.append("total_size", totalSize.toString());
formData.append("chunk", chunkData);
const response = await fetch(`${this.webroot}/upload-chunk`, {
method: "POST",
body: formData,
});
if (!response.ok) {
throw new Error(`Chunk ${chunkIndex} upload failed: ${response.status}`);
}
return response.json();
}
/**
* 取消上傳
*/
cancelUpload(uploadId) {
const upload = this.activeUploads.get(uploadId);
if (upload) {
upload.status = "cancelled";
this.activeUploads.delete(uploadId);
}
}
}
// ==================== 下載管理器 ====================
/**
* 下載管理器類別
*/
class DownloadManager {
/**
* @param {string} webroot - 網站根路徑
*/
constructor(webroot) {
this.webroot = webroot;
}
/**
* 下載檔案自動判斷使用直傳或 chunk
*
* @param {string} url - 下載 URL
* @param {string} fileName - 檔案名稱
* @param {object} options - 選項
* @param {function} options.onProgress - 進度回調
* @returns {Promise<Blob>} 下載的檔案
*/
async downloadFile(url, fileName, options = {}) {
const { onProgress } = options;
// 先取得檔案資訊
const info = await this.getFileInfo(url);
if (!info) {
// 無法取得資訊,使用直接下載
return this.downloadDirect(url, fileName);
}
if (shouldUseChunkedTransfer(info.total_size)) {
// 大檔:使用 chunk 下載
return this.downloadChunked(url, fileName, info, onProgress);
} else {
// 小檔:直接下載
return this.downloadDirect(url, fileName, onProgress);
}
}
/**
* 取得檔案資訊
*/
async getFileInfo(url) {
try {
const response = await fetch(`${url}/info`, { method: "GET" });
if (response.ok) {
return response.json();
}
} catch {
// 忽略錯誤,降級為直接下載
}
return null;
}
/**
* 直接下載小檔
*/
async downloadDirect(url, fileName, onProgress) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Download failed: ${response.status}`);
}
const contentLength = response.headers.get("Content-Length");
const total = contentLength ? parseInt(contentLength, 10) : 0;
const reader = response.body.getReader();
const chunks = [];
let loaded = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
loaded += value.length;
if (onProgress && total > 0) {
onProgress((loaded / total) * 100);
}
}
const blob = new Blob(chunks);
this.triggerDownload(blob, fileName);
return blob;
}
/**
* Chunk 下載大檔
*/
async downloadChunked(url, fileName, info, onProgress) {
const { total_chunks, chunk_size, total_size } = info;
const chunks = [];
for (let i = 0; i < total_chunks; i++) {
const chunkData = await this.downloadChunk(url, i);
chunks.push(chunkData);
if (onProgress) {
const loaded = Math.min((i + 1) * chunk_size, total_size);
onProgress((loaded / total_size) * 100);
}
}
const blob = new Blob(chunks);
this.triggerDownload(blob, fileName);
return blob;
}
/**
* 下載單個 chunk
*/
async downloadChunk(url, chunkIndex) {
const response = await fetch(`${url}/chunk/${chunkIndex}`);
if (!response.ok) {
throw new Error(`Chunk ${chunkIndex} download failed: ${response.status}`);
}
return response.arrayBuffer();
}
/**
* 觸發瀏覽器下載
*/
triggerDownload(blob, fileName) {
const url = URL.createObjectURL(blob);
const a = document.createElement("a");
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
}
/**
* 簡單下載不使用 chunk用於向後相容
*/
async simpleDownload(url, fileName) {
const a = document.createElement("a");
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
}
}
// ==================== 全域實例 ====================
// 從 meta 標籤取得 webroot
const webrootMeta = document.querySelector("meta[name='webroot']");
const webroot = webrootMeta ? webrootMeta.content : "";
// 建立全域實例
window.ContentsTransfer = {
uploadManager: new UploadManager(webroot),
downloadManager: new DownloadManager(webroot),
// 工具函數
shouldUseChunkedTransfer,
calculateChunkCount,
// 常數
CHUNK_THRESHOLD_BYTES,
CHUNK_SIZE_BYTES,
};
// 向後相容:提供簡化的 API
window.uploadFile = (file, options) => window.ContentsTransfer.uploadManager.uploadFile(file, options);
window.downloadFile = (url, fileName, options) => window.ContentsTransfer.downloadManager.downloadFile(url, fileName, options);
console.log("Contents.CN Transfer Module initialized");

View file

@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process";
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs"; import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs";
import { join, basename, dirname } from "node:path"; import { join, basename, dirname } from "node:path";
import { ExecFileFn } from "./types"; import { ExecFileFn } from "./types";
import { createConverterArchive, getArchiveFileName } from "../transfer";
export const properties = { export const properties = {
from: { from: {
@ -14,19 +15,22 @@ export const properties = {
}; };
/** /**
* Helper function to create a tar.gz archive from a directory * Helper function to create a .tar archive from a directory (no compression)
*
* 使 .tar .tar.gz / .tgz / .zip
*/ */
function createTarGzArchive( function createTarArchive(
sourceDir: string, sourceDir: string,
outputTarGz: string, outputTar: string,
execFile: ExecFileFn, execFile: ExecFileFn,
): Promise<void> { ): Promise<void> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
// Use tar command to create gzipped archive // Use tar command to create archive (without gzip compression)
// tar -czf <output.tar.gz> -C <sourceDir> . // tar -cf <output.tar> -C <sourceDir> .
// 注意:使用 -cf 而非 -czf避免 gzip 壓縮
execFile( execFile(
"tar", "tar",
["-czf", outputTarGz, "-C", sourceDir, "."], ["-cf", outputTar, "-C", sourceDir, "."],
(error, stdout, stderr) => { (error, stdout, stderr) => {
if (error) { if (error) {
reject(`tar error: ${error}`); reject(`tar error: ${error}`);
@ -122,15 +126,14 @@ export async function convert(
// MinerU outputs to a subdirectory, find the actual output // MinerU outputs to a subdirectory, find the actual output
const mineruActualOutput = join(mineruOutputDir, "auto"); const mineruActualOutput = join(mineruOutputDir, "auto");
// Create tar.gz archive from the output directory // Create .tar archive from the output directory (不使用壓縮)
const tarGzPath = targetPath.endsWith(".tar.gz") // 強制使用 .tar 格式,禁止 .tar.gz
? targetPath const tarPath = getArchiveFileName(targetPath);
: `${targetPath}.tar.gz`;
// Ensure the parent directory exists // Ensure the parent directory exists
const tarGzDir = dirname(tarGzPath); const tarDir = dirname(tarPath);
if (!existsSync(tarGzDir)) { if (!existsSync(tarDir)) {
mkdirSync(tarGzDir, { recursive: true }); mkdirSync(tarDir, { recursive: true });
} }
// Use the actual MinerU output directory for archiving // Use the actual MinerU output directory for archiving
@ -138,14 +141,14 @@ export async function convert(
? mineruActualOutput ? mineruActualOutput
: mineruOutputDir; : mineruOutputDir;
await createTarGzArchive(outputToArchive, tarGzPath, execFile); await createTarArchive(outputToArchive, tarPath, execFile);
// Clean up the temporary directory // Clean up the temporary directory
removeDir(mineruOutputDir); removeDir(mineruOutputDir);
resolve("Done"); resolve("Done");
} catch (tarError) { } catch (tarError) {
reject(`Failed to create tar.gz archive: ${tarError}`); reject(`Failed to create .tar archive: ${tarError}`);
} }
}); });
}); });

View file

@ -11,11 +11,13 @@ import { convert } from "./pages/convert";
import { deleteFile } from "./pages/deleteFile"; import { deleteFile } from "./pages/deleteFile";
import { deleteJob } from "./pages/deleteJob"; import { deleteJob } from "./pages/deleteJob";
import { download } from "./pages/download"; import { download } from "./pages/download";
import { downloadChunk } from "./pages/downloadChunk";
import { history } from "./pages/history"; import { history } from "./pages/history";
import { listConverters } from "./pages/listConverters"; import { listConverters } from "./pages/listConverters";
import { results } from "./pages/results"; import { results } from "./pages/results";
import { root } from "./pages/root"; import { root } from "./pages/root";
import { upload } from "./pages/upload"; import { upload } from "./pages/upload";
import { uploadChunk, uploadInfo } from "./pages/uploadChunk";
import { user } from "./pages/user"; import { user } from "./pages/user";
import { healthcheck } from "./pages/healthcheck"; import { healthcheck } from "./pages/healthcheck";
@ -41,9 +43,12 @@ const app = new Elysia({
.use(user) .use(user)
.use(root) .use(root)
.use(upload) .use(upload)
.use(uploadChunk)
.use(uploadInfo)
.use(history) .use(history)
.use(convert) .use(convert)
.use(download) .use(download)
.use(downloadChunk)
.use(deleteJob) .use(deleteJob)
.use(results) .use(results)
.use(deleteFile) .use(deleteFile)

View file

@ -1,11 +1,11 @@
import path from "node:path"; import path from "node:path";
import { Elysia } from "elysia"; import { Elysia } from "elysia";
import sanitize from "sanitize-filename"; import sanitize from "sanitize-filename";
import * as tar from "tar";
import { outputDir } from ".."; import { outputDir } from "..";
import db from "../db/db"; import db from "../db/db";
import { WEBROOT } from "../helpers/env"; import { WEBROOT } from "../helpers/env";
import { userService } from "./user"; import { userService } from "./user";
import { createJobArchive, shouldUseChunkedDownload } from "../transfer";
export const download = new Elysia() export const download = new Elysia()
.use(userService) .use(userService)
@ -45,18 +45,10 @@ export const download = new Elysia()
const jobId = decodeURIComponent(params.jobId); const jobId = decodeURIComponent(params.jobId);
const outputPath = `${outputDir}${userId}/${jobId}`; const outputPath = `${outputDir}${userId}/${jobId}`;
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
await tar.create( // 使用統一的封裝管理器建立 .tar不壓縮
{ const outputTar = await createJobArchive(outputPath, jobId);
file: outputTar,
cwd: outputPath,
filter: (path) => {
return !path.match(".*\\.tar");
},
},
["."],
);
return Bun.file(outputTar); return Bun.file(outputTar);
}, },
{ {

178
src/pages/downloadChunk.tsx Normal file
View file

@ -0,0 +1,178 @@
/**
* Contents.CN Chunk API
*
*
*/
import { Elysia, t } from "elysia";
import { join } from "node:path";
import { outputDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
createChunkDownloadHeaders,
getFileForDirectDownload,
createDirectDownloadHeaders,
CHUNK_SIZE_BYTES,
} from "../transfer";
import { statSync, existsSync } from "node:fs";
export const downloadChunk = new Elysia()
.use(userService)
/**
*
*/
.get(
"/download/:userId/:jobId/:fileName/info",
async ({ params, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const fileName = sanitize(decodeURIComponent(params.fileName));
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
const info = getChunkDownloadInfo(filePath);
if (!info) {
return { error: "File not found" };
}
return {
...info,
use_chunked: shouldUseChunkedDownload(filePath),
};
},
{ auth: true }
)
/**
* chunk
*/
.get(
"/download/:userId/:jobId/:fileName/chunk/:chunkIndex",
async ({ params, set, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
set.status = 404;
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const fileName = sanitize(decodeURIComponent(params.fileName));
const chunkIndex = parseInt(params.chunkIndex, 10);
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
const info = getChunkDownloadInfo(filePath);
if (!info) {
set.status = 404;
return { error: "File not found" };
}
const chunkData = await getChunk(filePath, chunkIndex);
if (!chunkData) {
set.status = 404;
return { error: "Chunk not found" };
}
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
for (const [key, value] of Object.entries(headers)) {
set.headers[key] = value;
}
return new Response(chunkData);
},
{ auth: true }
)
/**
* Archive chunk
*/
.get(
"/archive/:jobId/info",
async ({ params, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
if (!existsSync(archivePath)) {
return { error: "Archive not found" };
}
const info = getChunkDownloadInfo(archivePath);
if (!info) {
return { error: "Archive not found" };
}
return {
...info,
use_chunked: shouldUseChunkedDownload(archivePath),
};
},
{ auth: true }
)
/**
* Archive chunk
*/
.get(
"/archive/:jobId/chunk/:chunkIndex",
async ({ params, set, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
set.status = 404;
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const chunkIndex = parseInt(params.chunkIndex, 10);
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
const info = getChunkDownloadInfo(archivePath);
if (!info) {
set.status = 404;
return { error: "Archive not found" };
}
const chunkData = await getChunk(archivePath, chunkIndex);
if (!chunkData) {
set.status = 404;
return { error: "Chunk not found" };
}
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
for (const [key, value] of Object.entries(headers)) {
set.headers[key] = value;
}
return new Response(chunkData);
},
{ auth: true }
);

88
src/pages/uploadChunk.tsx Normal file
View file

@ -0,0 +1,88 @@
/**
* Contents.CN Chunk API
*
*
*/
import { Elysia, t } from "elysia";
import { WEBROOT } from "../helpers/env";
import { uploadsDir } from "../index";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import {
handleChunkUpload,
shouldUseChunkedUpload,
CHUNK_SIZE_BYTES,
calculateChunkCount,
} from "../transfer";
export const uploadChunk = new Elysia().use(userService).post(
"/upload-chunk",
async ({ body, user, cookie: { jobId } }) => {
if (!jobId?.value) {
return {
success: false,
message: "No active job session",
};
}
const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body;
const sanitizedFileName = sanitize(file_name);
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
// 取得 chunk 資料
const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk;
const result = await handleChunkUpload(
upload_id,
parseInt(chunk_index, 10),
parseInt(total_chunks, 10),
chunkData,
sanitizedFileName,
parseInt(total_size, 10),
user.id,
jobId.value,
`${uploadsDir}${user.id}/`,
userUploadsDir
);
return result;
},
{
body: t.Object({
upload_id: t.String(),
chunk_index: t.String(),
total_chunks: t.String(),
file_name: t.String(),
total_size: t.String(),
chunk: t.File(),
}),
auth: true,
}
);
/**
* chunk
*/
export const uploadInfo = new Elysia().use(userService).post(
"/upload-info",
async ({ body }) => {
const { file_size } = body;
const size = parseInt(file_size, 10);
const useChunked = shouldUseChunkedUpload(size);
return {
use_chunked: useChunked,
chunk_size: CHUNK_SIZE_BYTES,
total_chunks: useChunked ? calculateChunkCount(size) : 1,
};
},
{
body: t.Object({
file_size: t.String(),
}),
auth: true,
}
);

View file

@ -0,0 +1,154 @@
/**
* Contents.CN
*
*
* - .tar
* - .tar.gz, .tgz, .zip
*/
import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
import { join, basename } from "node:path";
import * as tar from "tar";
import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants";
/**
*
*/
export function validateArchiveFormat(fileName: string): boolean {
const lowerName = fileName.toLowerCase();
// 檢查是否使用禁止的格式
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
if (lowerName.endsWith(forbidden)) {
return false;
}
}
// 檢查是否是允許的格式
return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT);
}
/**
* 使 .tar
*/
export function getArchiveFileName(baseName: string): string {
const lowerName = baseName.toLowerCase();
// 移除任何禁止的副檔名
let cleanName = baseName;
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
if (lowerName.endsWith(forbidden)) {
cleanName = baseName.slice(0, -forbidden.length);
break;
}
}
// 如果已經是 .tar 結尾,直接返回
if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) {
return cleanName;
}
// 否則加上 .tar
return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`;
}
/**
* .tar
*
* @param sourceDir -
* @param outputPath - .tar
* @param options -
*/
export async function createTarArchive(
sourceDir: string,
outputPath: string,
options: {
filter?: (path: string) => boolean;
prefix?: string;
} = {}
): Promise<string> {
// 強制確保輸出是 .tar 格式
const finalOutputPath = getArchiveFileName(outputPath);
// 確保輸出目錄存在
const outputDir = join(finalOutputPath, "..");
if (!existsSync(outputDir)) {
mkdirSync(outputDir, { recursive: true });
}
// 取得要封裝的檔案列表
const files = readdirSync(sourceDir);
// 預設過濾器:排除其他 tar 檔案
const defaultFilter = (path: string) => !path.match(/\.tar$/i);
const filter = options.filter || defaultFilter;
await tar.create(
{
file: finalOutputPath,
cwd: sourceDir,
filter: filter,
// 不使用 gzip 壓縮
gzip: false,
},
files.filter(f => filter(f))
);
console.log(`Created tar archive: ${finalOutputPath}`);
return finalOutputPath;
}
/**
* .tar
*
* @param outputDir -
* @param jobId - ID
*/
export async function createJobArchive(outputDir: string, jobId: string): Promise<string> {
const archiveName = `converted_files_${jobId}.tar`;
const archivePath = join(outputDir, archiveName);
await createTarArchive(outputDir, archivePath, {
filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案
});
return archivePath;
}
/**
* MinerU
*
* @param mineruOutputDir - MinerU
* @param targetPath - .tar
*/
export async function createConverterArchive(
converterOutputDir: string,
targetPath: string
): Promise<string> {
// 確保輸出使用 .tar 格式
const tarPath = getArchiveFileName(targetPath);
await createTarArchive(converterOutputDir, tarPath);
return tarPath;
}
/**
*
*/
export function getArchiveInfo(archivePath: string): {
exists: boolean;
size: number;
fileName: string;
} | null {
if (!existsSync(archivePath)) {
return null;
}
const stats = statSync(archivePath);
return {
exists: true,
size: stats.size,
fileName: basename(archivePath),
};
}

38
src/transfer/constants.ts Normal file
View file

@ -0,0 +1,38 @@
/**
* Contents.CN
*
*
*/
/**
* 10MB
* -
* - 使 chunk
*/
export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
/**
* chunk 5MB
*/
export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
/**
* session
* 1
*/
export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
/**
*
*/
export const CHUNK_TEMP_DIR = "chunks_temp";
/**
* .tar
*/
export const ALLOWED_ARCHIVE_FORMAT = ".tar";
/**
*
*/
export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"];

View file

@ -0,0 +1,123 @@
/**
* Contents.CN
*
*
* - 10MB
* - >10MBchunk
*/
import { existsSync, statSync, createReadStream } from "node:fs";
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants";
import type { ChunkDownloadInfo } from "./types";
import { getTransferMode } from "./types";
import { basename } from "node:path";
/**
* 使 chunk
*/
export function shouldUseChunkedDownload(filePath: string): boolean {
if (!existsSync(filePath)) {
return false;
}
const stats = statSync(filePath);
return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked";
}
/**
* chunk
*/
export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null {
if (!existsSync(filePath)) {
return null;
}
const stats = statSync(filePath);
const fileName = basename(filePath);
return {
total_size: stats.size,
total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES),
chunk_size: CHUNK_SIZE_BYTES,
file_name: fileName,
};
}
/**
* chunk
*/
export async function getChunk(
filePath: string,
chunkIndex: number,
chunkSize: number = CHUNK_SIZE_BYTES
): Promise<Buffer | null> {
if (!existsSync(filePath)) {
return null;
}
const stats = statSync(filePath);
const start = chunkIndex * chunkSize;
const end = Math.min(start + chunkSize, stats.size);
if (start >= stats.size) {
return null;
}
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
const stream = createReadStream(filePath, { start, end: end - 1 });
stream.on("data", (chunk) => {
chunks.push(chunk as Buffer);
});
stream.on("end", () => {
resolve(Buffer.concat(chunks));
});
stream.on("error", reject);
});
}
/**
*
*/
export function getFileForDirectDownload(filePath: string): ReturnType<typeof Bun.file> | null {
if (!existsSync(filePath)) {
return null;
}
return Bun.file(filePath);
}
/**
* chunk headers
*/
export function createChunkDownloadHeaders(
info: ChunkDownloadInfo,
chunkIndex: number,
chunkData: Buffer
): Record<string, string> {
const start = chunkIndex * info.chunk_size;
const end = start + chunkData.length - 1;
return {
"Content-Type": "application/octet-stream",
"Content-Length": chunkData.length.toString(),
"Content-Range": `bytes ${start}-${end}/${info.total_size}`,
"Accept-Ranges": "bytes",
"X-Chunk-Index": chunkIndex.toString(),
"X-Total-Chunks": info.total_chunks.toString(),
"X-File-Name": encodeURIComponent(info.file_name),
"X-Total-Size": info.total_size.toString(),
};
}
/**
* headers
*/
export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record<string, string> {
return {
"Content-Type": "application/octet-stream",
"Content-Length": fileSize.toString(),
"Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`,
};
}

56
src/transfer/index.ts Normal file
View file

@ -0,0 +1,56 @@
/**
* Contents.CN
*
*
*/
// 常數
export {
CHUNK_THRESHOLD_BYTES,
CHUNK_SIZE_BYTES,
UPLOAD_SESSION_TIMEOUT_MS,
CHUNK_TEMP_DIR,
ALLOWED_ARCHIVE_FORMAT,
FORBIDDEN_ARCHIVE_FORMATS,
} from "./constants";
// 類型
export type {
ChunkUploadRequest,
ChunkUploadResponse,
DirectUploadResponse,
ChunkDownloadInfo,
UploadSession,
TransferMode,
} from "./types";
export { getTransferMode } from "./types";
// 上傳管理
export {
uploadSessionManager,
shouldUseChunkedUpload,
handleDirectUpload,
handleChunkUpload,
calculateChunkCount,
} from "./uploadManager";
// 下載管理
export {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
getFileForDirectDownload,
createChunkDownloadHeaders,
createDirectDownloadHeaders,
} from "./downloadManager";
// 封裝管理
export {
validateArchiveFormat,
getArchiveFileName,
createTarArchive,
createJobArchive,
createConverterArchive,
getArchiveInfo,
} from "./archiveManager";

94
src/transfer/types.ts Normal file
View file

@ -0,0 +1,94 @@
/**
* Contents.CN
*/
/**
* Chunk
*/
export interface ChunkUploadRequest {
/** 上傳會話 IDUUID */
upload_id: string;
/** 當前 chunk 索引(從 0 開始) */
chunk_index: number;
/** 總 chunk 數量 */
total_chunks: number;
/** chunk 數據 */
data: Blob | ArrayBuffer;
/** 原始檔案名稱 */
file_name: string;
/** 檔案總大小 */
total_size: number;
}
/**
* Chunk
*/
export interface ChunkUploadResponse {
success: boolean;
message: string;
/** 已接收的 chunks 索引列表 */
received_chunks?: number[];
/** 是否所有 chunks 都已接收完成 */
completed?: boolean;
/** 合併後的檔案路徑(僅完成時返回) */
file_path?: string;
}
/**
*
*/
export interface DirectUploadResponse {
success: boolean;
message: string;
file_path?: string;
}
/**
* Chunk
*/
export interface ChunkDownloadInfo {
/** 檔案總大小 */
total_size: number;
/** chunk 數量 */
total_chunks: number;
/** 每個 chunk 大小 */
chunk_size: number;
/** 檔案名稱 */
file_name: string;
}
/**
*
*/
export interface UploadSession {
/** 上傳 ID */
upload_id: string;
/** 使用者 ID */
user_id: string;
/** Job ID */
job_id: string;
/** 檔案名稱 */
file_name: string;
/** 檔案總大小 */
total_size: number;
/** 總 chunk 數 */
total_chunks: number;
/** 已接收的 chunks */
received_chunks: Set<number>;
/** 建立時間 */
created_at: Date;
/** 暫存目錄路徑 */
temp_dir: string;
}
/**
*
*/
export type TransferMode = "direct" | "chunked";
/**
*
*/
export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode {
return fileSize <= thresholdBytes ? "direct" : "chunked";
}

View file

@ -0,0 +1,303 @@
/**
* Contents.CN
*
*
* - 10MB
* - >10MBchunk
*/
import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs";
import { join } from "node:path";
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants";
import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types";
import { getTransferMode } from "./types";
/**
*
* chunk
*/
class UploadSessionManager {
private sessions: Map<string, UploadSession> = new Map();
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor() {
// 定期清理過期的 sessions
this.cleanupInterval = setInterval(() => {
this.cleanupExpiredSessions();
}, 5 * 60 * 1000); // 每 5 分鐘清理一次
}
/**
*
*/
createSession(
uploadId: string,
userId: string,
jobId: string,
fileName: string,
totalSize: number,
totalChunks: number,
baseTempDir: string
): UploadSession {
const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId);
if (!existsSync(tempDir)) {
mkdirSync(tempDir, { recursive: true });
}
const session: UploadSession = {
upload_id: uploadId,
user_id: userId,
job_id: jobId,
file_name: fileName,
total_size: totalSize,
total_chunks: totalChunks,
received_chunks: new Set(),
created_at: new Date(),
temp_dir: tempDir,
};
this.sessions.set(uploadId, session);
return session;
}
/**
*
*/
getSession(uploadId: string): UploadSession | undefined {
return this.sessions.get(uploadId);
}
/**
* chunk
*/
markChunkReceived(uploadId: string, chunkIndex: number): boolean {
const session = this.sessions.get(uploadId);
if (!session) return false;
session.received_chunks.add(chunkIndex);
return true;
}
/**
* chunks
*/
isComplete(uploadId: string): boolean {
const session = this.sessions.get(uploadId);
if (!session) return false;
return session.received_chunks.size === session.total_chunks;
}
/**
*
*/
removeSession(uploadId: string): void {
const session = this.sessions.get(uploadId);
if (session && existsSync(session.temp_dir)) {
rmSync(session.temp_dir, { recursive: true, force: true });
}
this.sessions.delete(uploadId);
}
/**
*
*/
private cleanupExpiredSessions(): void {
const now = Date.now();
for (const [uploadId, session] of this.sessions) {
if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) {
console.log(`Cleaning up expired upload session: ${uploadId}`);
this.removeSession(uploadId);
}
}
}
/**
*
*/
destroy(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = null;
}
}
}
// 單例實例
export const uploadSessionManager = new UploadSessionManager();
/**
* 使 chunk
*/
export function shouldUseChunkedUpload(fileSize: number): boolean {
return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked";
}
/**
*
*/
export async function handleDirectUpload(
file: File,
targetDir: string,
fileName: string
): Promise<DirectUploadResponse> {
try {
const targetPath = join(targetDir, fileName);
if (!existsSync(targetDir)) {
mkdirSync(targetDir, { recursive: true });
}
await Bun.write(targetPath, file);
return {
success: true,
message: "File uploaded successfully.",
file_path: targetPath,
};
} catch (error) {
console.error("Direct upload failed:", error);
return {
success: false,
message: `Upload failed: ${error}`,
};
}
}
/**
* chunk
*/
export async function handleChunkUpload(
uploadId: string,
chunkIndex: number,
totalChunks: number,
chunkData: ArrayBuffer | Blob | Buffer,
fileName: string,
totalSize: number,
userId: string,
jobId: string,
baseTempDir: string,
targetDir: string
): Promise<ChunkUploadResponse> {
try {
// 取得或建立會話
let session = uploadSessionManager.getSession(uploadId);
if (!session) {
session = uploadSessionManager.createSession(
uploadId,
userId,
jobId,
fileName,
totalSize,
totalChunks,
baseTempDir
);
}
// 驗證會話資訊一致性
if (session.total_chunks !== totalChunks || session.file_name !== fileName) {
return {
success: false,
message: "Session mismatch: inconsistent upload parameters",
};
}
// 儲存 chunk
const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`);
// 處理不同類型的 chunk 資料
let data: ArrayBuffer | Buffer;
if (chunkData instanceof Blob) {
data = await chunkData.arrayBuffer();
} else if (Buffer.isBuffer(chunkData)) {
data = chunkData;
} else {
data = chunkData;
}
await Bun.write(chunkPath, data);
// 標記已接收
uploadSessionManager.markChunkReceived(uploadId, chunkIndex);
// 檢查是否完成
if (uploadSessionManager.isComplete(uploadId)) {
// 合併所有 chunks
const finalPath = await mergeChunks(session, targetDir);
// 清理會話
uploadSessionManager.removeSession(uploadId);
return {
success: true,
message: "Upload completed and merged successfully.",
received_chunks: Array.from({ length: totalChunks }, (_, i) => i),
completed: true,
file_path: finalPath,
};
}
return {
success: true,
message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`,
received_chunks: Array.from(session.received_chunks),
completed: false,
};
} catch (error) {
console.error("Chunk upload failed:", error);
return {
success: false,
message: `Chunk upload failed: ${error}`,
};
}
}
/**
* chunks
*/
async function mergeChunks(session: UploadSession, targetDir: string): Promise<string> {
const targetPath = join(targetDir, session.file_name);
if (!existsSync(targetDir)) {
mkdirSync(targetDir, { recursive: true });
}
// 讀取並排序所有 chunks
const chunkFiles = readdirSync(session.temp_dir)
.filter(f => f.startsWith("chunk_"))
.sort();
// 建立輸出串流
const writeStream = createWriteStream(targetPath);
// 依序寫入每個 chunk
for (const chunkFile of chunkFiles) {
const chunkPath = join(session.temp_dir, chunkFile);
const chunkData = await Bun.file(chunkPath).arrayBuffer();
writeStream.write(Buffer.from(chunkData));
}
// 結束寫入
await new Promise<void>((resolve, reject) => {
writeStream.end((err: Error | null) => {
if (err) reject(err);
else resolve();
});
});
// 驗證檔案大小
const finalStats = statSync(targetPath);
if (finalStats.size !== session.total_size) {
console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`);
}
console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`);
return targetPath;
}
/**
* chunk
*/
export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
return Math.ceil(fileSize / chunkSize);
}

View file

@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => {
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |"); writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
callback(null, "MinerU conversion complete", ""); callback(null, "MinerU conversion complete", "");
} else if (cmd === "tar") { } else if (cmd === "tar") {
// Simulate tar.gz creation // Simulate .tar creation (no compression)
callback(null, "Archive created", ""); callback(null, "Archive created", "");
} }
}; };
const targetPath = join(testDir, "output.tar.gz"); // 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "output.tar");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(mineruArgs).toContain("--table-mode"); expect(mineruArgs).toContain("--table-mode");
@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => {
writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)"); writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)");
callback(null, "MinerU conversion complete", ""); callback(null, "MinerU conversion complete", "");
} else if (cmd === "tar") { } else if (cmd === "tar") {
// Simulate tar.gz creation // Simulate .tar creation (no compression)
callback(null, "Archive created", ""); callback(null, "Archive created", "");
} }
}; };
const targetPath = join(testDir, "output.tar.gz"); // 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "output.tar");
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile); await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
expect(capturedArgs).toContain("--table-mode"); expect(capturedArgs).toContain("--table-mode");
@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => {
}); });
}); });
describe("MinerU converter tar.gz output", () => { describe("MinerU converter .tar output (no compression)", () => {
const testDir = "./test-output-mineru-targz"; const testDir = "./test-output-mineru-tar";
beforeEach(() => { beforeEach(() => {
if (!existsSync(testDir)) { if (!existsSync(testDir)) {
@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => {
} }
}); });
test("should create tar.gz archive from output", async () => { test("should create .tar archive from output (without gzip)", async () => {
let tarCalled = false; let tarCalled = false;
let tarArgs: string[] = []; let tarArgs: string[] = [];
@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => {
} }
}; };
const targetPath = join(testDir, "output.tar.gz"); // 使用 .tar 格式
const targetPath = join(testDir, "output.tar");
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(result).toBe("Done"); expect(result).toBe("Done");
expect(tarCalled).toBe(true); expect(tarCalled).toBe(true);
expect(tarArgs).toContain("-czf"); // 驗證使用 -cf 而非 -czf不壓縮
expect(tarArgs).toContain("-cf");
// 確保沒有使用 gzip 壓縮標誌
expect(tarArgs).not.toContain("-czf");
}); });
test("should reject on mineru error", async () => { test("should reject on mineru error", async () => {
@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => {
} }
}; };
const targetPath = join(testDir, "output.tar.gz"); const targetPath = join(testDir, "output.tar");
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile)) expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
.rejects.toMatch(/mineru error/); .rejects.toMatch(/mineru error/);
}); });
test("should use correct tar arguments for compression", async () => { test("should use correct tar arguments without compression", async () => {
let tarArgs: string[] = []; let tarArgs: string[] = [];
const mockExecFile: ExecFileFn = ( const mockExecFile: ExecFileFn = (
@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => {
} }
}; };
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz"); // 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "test_MINERU_md-t.tar");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
// Verify tar is called with correct compression flags // 驗證 tar 使用正確的無壓縮標誌
expect(tarArgs[0]).toBe("-czf"); expect(tarArgs[0]).toBe("-cf");
expect(tarArgs[1]).toContain(".tar.gz"); expect(tarArgs[1]).toContain(".tar");
expect(tarArgs[1]).not.toContain(".tar.gz");
expect(tarArgs[2]).toBe("-C"); expect(tarArgs[2]).toBe("-C");
expect(tarArgs[4]).toBe("."); expect(tarArgs[4]).toBe(".");
}); });
test("should convert .tar.gz target path to .tar", async () => {
let tarOutputPath = "";
const mockExecFile: ExecFileFn = (
cmd: string,
args: string[],
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "mineru") {
const outputDir = args[3];
if (outputDir && !existsSync(outputDir)) {
mkdirSync(outputDir, { recursive: true });
}
const autoDir = join(outputDir, "auto");
if (!existsSync(autoDir)) {
mkdirSync(autoDir, { recursive: true });
}
writeFileSync(join(autoDir, "output.md"), "# Test");
callback(null, "Done", "");
} else if (cmd === "tar") {
tarOutputPath = args[1]; // 輸出路徑
callback(null, "Archive created", "");
}
};
// 即使傳入 .tar.gz也應該被轉換為 .tar
const targetPath = join(testDir, "output.tar.gz");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
// 驗證輸出是 .tar 而非 .tar.gz
expect(tarOutputPath).toContain(".tar");
expect(tarOutputPath).not.toContain(".tar.gz");
});
}); });

View file

@ -0,0 +1,208 @@
/**
* Contents.CN Chunk
*
* chunk
*/
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
import { mkdirSync, existsSync, writeFileSync, rmSync } from "node:fs";
import { join } from "node:path";
import {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
createChunkDownloadHeaders,
} from "../../src/transfer/downloadManager";
import { CHUNK_SIZE_BYTES, CHUNK_THRESHOLD_BYTES } from "../../src/transfer/constants";
const testDir = "./test-output-chunk-download";
describe("Chunk 下載資訊測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("getChunkDownloadInfo 應返回正確的下載資訊", () => {
const testFile = join(testDir, "large-file.bin");
const fileSize = 25 * 1024 * 1024; // 25MB
// 建立一個假檔案(只寫入少量資料模擬)
const content = Buffer.alloc(1024, "X"); // 1KB
writeFileSync(testFile, content);
const info = getChunkDownloadInfo(testFile);
expect(info).not.toBeNull();
expect(info!.file_name).toBe("large-file.bin");
expect(info!.total_size).toBe(1024);
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
expect(info!.total_chunks).toBe(1); // 1KB 只需要 1 chunk
});
test("不存在的檔案應返回 null", () => {
const info = getChunkDownloadInfo(join(testDir, "non-existent.bin"));
expect(info).toBeNull();
});
});
describe("Chunk 讀取測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("getChunk 應正確讀取指定的 chunk", async () => {
const testFile = join(testDir, "chunked-file.bin");
// 建立測試檔案20 bytes分成 4 個 5-byte chunks
const content = "AAAAABBBBBCCCCCDDDD"; // 19 bytes
writeFileSync(testFile, content);
// 讀取第一個 chunk5 bytes
const chunk0 = await getChunk(testFile, 0, 5);
expect(chunk0!.toString()).toBe("AAAAA");
// 讀取第二個 chunk
const chunk1 = await getChunk(testFile, 1, 5);
expect(chunk1!.toString()).toBe("BBBBB");
// 讀取第三個 chunk
const chunk2 = await getChunk(testFile, 2, 5);
expect(chunk2!.toString()).toBe("CCCCC");
// 讀取最後一個 chunk只有 4 bytes
const chunk3 = await getChunk(testFile, 3, 5);
expect(chunk3!.toString()).toBe("DDDD");
});
test("讀取超出範圍的 chunk 應返回 null", async () => {
const testFile = join(testDir, "small-file.txt");
writeFileSync(testFile, "Hello");
const chunk = await getChunk(testFile, 10, 5); // 超出範圍
expect(chunk).toBeNull();
});
test("不存在的檔案應返回 null", async () => {
const chunk = await getChunk(join(testDir, "non-existent.bin"), 0, 5);
expect(chunk).toBeNull();
});
});
describe("Chunk 下載 Headers 測試", () => {
test("createChunkDownloadHeaders 應生成正確的 headers", () => {
const info = {
total_size: 1000,
total_chunks: 4,
chunk_size: 256,
file_name: "test-file.bin",
};
const chunkData = Buffer.alloc(256, "X");
const headers = createChunkDownloadHeaders(info, 0, chunkData);
expect(headers["Content-Type"]).toBe("application/octet-stream");
expect(headers["Content-Length"]).toBe("256");
expect(headers["Content-Range"]).toBe("bytes 0-255/1000");
expect(headers["Accept-Ranges"]).toBe("bytes");
expect(headers["X-Chunk-Index"]).toBe("0");
expect(headers["X-Total-Chunks"]).toBe("4");
expect(headers["X-Total-Size"]).toBe("1000");
});
test("最後一個 chunk 的 headers 應正確", () => {
const info = {
total_size: 1000,
total_chunks: 4,
chunk_size: 256,
file_name: "test-file.bin",
};
// 最後一個 chunk 只有 232 bytes (1000 - 256*3)
const chunkData = Buffer.alloc(232, "Z");
const headers = createChunkDownloadHeaders(info, 3, chunkData);
expect(headers["Content-Length"]).toBe("232");
expect(headers["Content-Range"]).toBe("bytes 768-999/1000");
expect(headers["X-Chunk-Index"]).toBe("3");
});
});
describe("傳輸模式判斷測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("小檔≤10MB不應使用 chunk 下載", () => {
const smallFile = join(testDir, "small.txt");
writeFileSync(smallFile, "Small file content");
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
});
test("不存在的檔案不應使用 chunk 下載", () => {
expect(shouldUseChunkedDownload(join(testDir, "non-existent.bin"))).toBe(false);
});
});
describe("完整 chunk 下載流程測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("應能完整讀取並合併所有 chunks", async () => {
const testFile = join(testDir, "complete-file.bin");
const originalContent = "AAAAAABBBBBBCCCCCCDDDDDDEEEEEE"; // 30 bytes
writeFileSync(testFile, originalContent);
const info = getChunkDownloadInfo(testFile);
expect(info).not.toBeNull();
// 模擬分段讀取(每個 chunk 10 bytes
const chunkSize = 10;
const totalChunks = Math.ceil(originalContent.length / chunkSize);
const chunks: Buffer[] = [];
for (let i = 0; i < totalChunks; i++) {
const chunk = await getChunk(testFile, i, chunkSize);
if (chunk) {
chunks.push(chunk);
}
}
// 合併所有 chunks
const mergedContent = Buffer.concat(chunks).toString();
expect(mergedContent).toBe(originalContent);
});
});

View file

@ -0,0 +1,178 @@
/**
* Contents.CN Chunk
*
* chunk
*/
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import {
handleChunkUpload,
uploadSessionManager,
calculateChunkCount,
} from "../../src/transfer/uploadManager";
import { CHUNK_SIZE_BYTES } from "../../src/transfer/constants";
const testDir = "./test-output-chunk-upload";
describe("Chunk 上傳整合測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("多 chunk 上傳應正確合併", async () => {
const uploadId = "test-chunked-upload-1";
const fileName = "large-file.bin";
const userId = "test-user";
const jobId = "test-job";
// 建立測試資料(模擬 15MB 檔案,分 3 個 chunks
const chunkSize = 5 * 1024; // 使用較小的 chunk 以便測試
const chunk1 = Buffer.alloc(chunkSize, "A");
const chunk2 = Buffer.alloc(chunkSize, "B");
const chunk3 = Buffer.alloc(chunkSize, "C");
const totalSize = chunkSize * 3;
const totalChunks = 3;
const baseTempDir = join(testDir, "temp");
const targetDir = join(testDir, "uploads");
mkdirSync(baseTempDir, { recursive: true });
mkdirSync(targetDir, { recursive: true });
// 上傳 chunk 1
const result1 = await handleChunkUpload(
uploadId, 0, totalChunks, chunk1, fileName, totalSize,
userId, jobId, baseTempDir, targetDir
);
expect(result1.success).toBe(true);
expect(result1.completed).toBe(false);
expect(result1.received_chunks).toContain(0);
// 上傳 chunk 2
const result2 = await handleChunkUpload(
uploadId, 1, totalChunks, chunk2, fileName, totalSize,
userId, jobId, baseTempDir, targetDir
);
expect(result2.success).toBe(true);
expect(result2.completed).toBe(false);
// 上傳 chunk 3最後一個
const result3 = await handleChunkUpload(
uploadId, 2, totalChunks, chunk3, fileName, totalSize,
userId, jobId, baseTempDir, targetDir
);
expect(result3.success).toBe(true);
expect(result3.completed).toBe(true);
expect(result3.file_path).toBeDefined();
// 驗證合併後的檔案
const mergedFile = join(targetDir, fileName);
expect(existsSync(mergedFile)).toBe(true);
const mergedContent = readFileSync(mergedFile);
expect(mergedContent.length).toBe(totalSize);
// 驗證內容正確性
const expectedContent = Buffer.concat([chunk1, chunk2, chunk3]);
expect(mergedContent.equals(expectedContent)).toBe(true);
});
test("亂序上傳 chunks 也應正確合併", async () => {
const uploadId = "test-chunked-upload-2";
const fileName = "out-of-order.bin";
const userId = "test-user";
const jobId = "test-job";
const chunkSize = 1024;
const chunk0 = Buffer.alloc(chunkSize, "0");
const chunk1 = Buffer.alloc(chunkSize, "1");
const chunk2 = Buffer.alloc(chunkSize, "2");
const chunk3 = Buffer.alloc(chunkSize, "3");
const totalSize = chunkSize * 4;
const totalChunks = 4;
const baseTempDir = join(testDir, "temp2");
const targetDir = join(testDir, "uploads2");
mkdirSync(baseTempDir, { recursive: true });
mkdirSync(targetDir, { recursive: true });
// 亂序上傳3, 1, 0, 2
await handleChunkUpload(uploadId, 3, totalChunks, chunk3, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
const finalResult = await handleChunkUpload(uploadId, 2, totalChunks, chunk2, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
expect(finalResult.success).toBe(true);
expect(finalResult.completed).toBe(true);
// 驗證檔案順序正確
const mergedFile = join(targetDir, fileName);
const mergedContent = readFileSync(mergedFile);
// 即使亂序上傳,合併後應該按正確順序
const expectedContent = Buffer.concat([chunk0, chunk1, chunk2, chunk3]);
expect(mergedContent.equals(expectedContent)).toBe(true);
});
test("重複上傳相同 chunk 應被處理", async () => {
const uploadId = "test-chunked-upload-3";
const fileName = "duplicate-chunk.bin";
const userId = "test-user";
const jobId = "test-job";
const chunkSize = 512;
const chunk0 = Buffer.alloc(chunkSize, "X");
const chunk1 = Buffer.alloc(chunkSize, "Y");
const totalSize = chunkSize * 2;
const totalChunks = 2;
const baseTempDir = join(testDir, "temp3");
const targetDir = join(testDir, "uploads3");
mkdirSync(baseTempDir, { recursive: true });
mkdirSync(targetDir, { recursive: true });
// 上傳 chunk 0
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
// 重複上傳 chunk 0
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
// 上傳 chunk 1
const result = await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
expect(result.success).toBe(true);
expect(result.completed).toBe(true);
// 驗證檔案大小正確(不應該因為重複上傳而變大)
const mergedFile = join(targetDir, fileName);
const stats = statSync(mergedFile);
expect(stats.size).toBe(totalSize);
});
});
describe("Chunk 數量計算測試", () => {
test("應正確計算不同大小檔案的 chunk 數量", () => {
const chunkSize = CHUNK_SIZE_BYTES;
// 剛好 1 個 chunk
expect(calculateChunkCount(chunkSize)).toBe(1);
// 多 1 byte 需要 2 個 chunks
expect(calculateChunkCount(chunkSize + 1)).toBe(2);
// 50MB 需要 10 個 chunks
expect(calculateChunkCount(50 * 1024 * 1024)).toBe(10);
// 1 byte 需要 1 個 chunk
expect(calculateChunkCount(1)).toBe(1);
});
});

View file

@ -0,0 +1,331 @@
/**
* Contents.CN
*
*
* 1. 10MB
* 2. chunk >10MB
* 3. chunk
* 4. .tar
* 5. End-to-End
*/
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import {
shouldUseChunkedUpload,
calculateChunkCount,
handleDirectUpload,
handleChunkUpload,
uploadSessionManager,
} from "../../src/transfer/uploadManager";
import {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
} from "../../src/transfer/downloadManager";
import {
validateArchiveFormat,
getArchiveFileName,
createTarArchive,
} from "../../src/transfer/archiveManager";
import {
CHUNK_THRESHOLD_BYTES,
CHUNK_SIZE_BYTES,
ALLOWED_ARCHIVE_FORMAT,
FORBIDDEN_ARCHIVE_FORMATS,
} from "../../src/transfer/constants";
import { getTransferMode } from "../../src/transfer/types";
const testDir = "./test-output-transfer";
describe("傳輸常數測試", () => {
test("CHUNK_THRESHOLD_BYTES 應為 10MB", () => {
expect(CHUNK_THRESHOLD_BYTES).toBe(10 * 1024 * 1024);
});
test("CHUNK_SIZE_BYTES 應為 5MB", () => {
expect(CHUNK_SIZE_BYTES).toBe(5 * 1024 * 1024);
});
test("唯一允許的封裝格式應為 .tar", () => {
expect(ALLOWED_ARCHIVE_FORMAT).toBe(".tar");
});
test("禁止的封裝格式應包含 .tar.gz, .tgz, .zip, .gz", () => {
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tar.gz");
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tgz");
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".zip");
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".gz");
});
});
describe("傳輸模式判斷測試", () => {
test("≤10MB 應使用直傳模式", () => {
expect(getTransferMode(10 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
expect(getTransferMode(5 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
expect(getTransferMode(1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
expect(getTransferMode(0, CHUNK_THRESHOLD_BYTES)).toBe("direct");
});
test(">10MB 應使用 chunk 模式", () => {
expect(getTransferMode(10 * 1024 * 1024 + 1, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
expect(getTransferMode(50 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
expect(getTransferMode(100 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
});
test("shouldUseChunkedUpload 函數應正確判斷", () => {
expect(shouldUseChunkedUpload(10 * 1024 * 1024)).toBe(false);
expect(shouldUseChunkedUpload(10 * 1024 * 1024 + 1)).toBe(true);
});
});
describe("Chunk 數量計算測試", () => {
test("計算 chunk 數量應正確", () => {
// 10MB / 5MB = 2 chunks
expect(calculateChunkCount(10 * 1024 * 1024)).toBe(2);
// 15MB / 5MB = 3 chunks
expect(calculateChunkCount(15 * 1024 * 1024)).toBe(3);
// 1 byte 也需要 1 chunk
expect(calculateChunkCount(1)).toBe(1);
// 5MB + 1 byte = 2 chunks
expect(calculateChunkCount(5 * 1024 * 1024 + 1)).toBe(2);
});
});
describe("封裝格式驗證測試", () => {
test("validateArchiveFormat 應正確驗證格式", () => {
// 允許的格式
expect(validateArchiveFormat("output.tar")).toBe(true);
expect(validateArchiveFormat("my_archive.tar")).toBe(true);
// 禁止的格式
expect(validateArchiveFormat("output.tar.gz")).toBe(false);
expect(validateArchiveFormat("output.tgz")).toBe(false);
expect(validateArchiveFormat("output.zip")).toBe(false);
expect(validateArchiveFormat("output.gz")).toBe(false);
});
test("getArchiveFileName 應強制轉換為 .tar", () => {
// 已經是 .tar
expect(getArchiveFileName("output.tar")).toBe("output.tar");
// 轉換 .tar.gz -> .tar
expect(getArchiveFileName("output.tar.gz")).toBe("output.tar");
// 轉換 .tgz -> .tar
expect(getArchiveFileName("output.tgz")).toBe("output.tar");
// 無副檔名 -> 加上 .tar
expect(getArchiveFileName("output")).toBe("output.tar");
// .zip 也是禁止的格式,會被移除並加上 .tar
expect(getArchiveFileName("output.zip")).toBe("output.tar");
// .gz 也是禁止的格式
expect(getArchiveFileName("output.gz")).toBe("output.tar");
});
});
describe("上傳管理測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("小檔直傳應成功", async () => {
const targetDir = join(testDir, "uploads");
const testContent = "Hello, World!";
const file = new File([testContent], "test.txt", { type: "text/plain" });
const result = await handleDirectUpload(file as any, targetDir, "test.txt");
expect(result.success).toBe(true);
expect(existsSync(join(targetDir, "test.txt"))).toBe(true);
expect(readFileSync(join(targetDir, "test.txt"), "utf-8")).toBe(testContent);
});
});
describe("下載管理測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("小檔不應使用 chunk 下載", () => {
const smallFile = join(testDir, "small.txt");
writeFileSync(smallFile, "Small content");
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
});
test("getChunkDownloadInfo 應返回正確資訊", () => {
const testFile = join(testDir, "test.txt");
const content = "Test content for download";
writeFileSync(testFile, content);
const info = getChunkDownloadInfo(testFile);
expect(info).not.toBeNull();
expect(info!.file_name).toBe("test.txt");
expect(info!.total_size).toBe(content.length);
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
});
test("getChunk 應正確讀取 chunk 資料", async () => {
const testFile = join(testDir, "test.txt");
const content = "ABCDEFGHIJ"; // 10 bytes
writeFileSync(testFile, content);
// 讀取第一個 chunk整個檔案因為小於 chunk size
const chunk = await getChunk(testFile, 0, 5);
expect(chunk).not.toBeNull();
expect(chunk!.toString()).toBe("ABCDE");
// 讀取第二個 chunk
const chunk2 = await getChunk(testFile, 1, 5);
expect(chunk2!.toString()).toBe("FGHIJ");
});
});
describe(".tar 封裝測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("createTarArchive 應建立 .tar 檔案", async () => {
const sourceDir = join(testDir, "source");
const outputTar = join(testDir, "output.tar");
mkdirSync(sourceDir, { recursive: true });
writeFileSync(join(sourceDir, "file1.txt"), "Content 1");
writeFileSync(join(sourceDir, "file2.txt"), "Content 2");
const result = await createTarArchive(sourceDir, outputTar);
expect(existsSync(result)).toBe(true);
expect(result.endsWith(".tar")).toBe(true);
expect(result).not.toContain(".tar.gz");
});
test("createTarArchive 應強制使用 .tar 格式", async () => {
const sourceDir = join(testDir, "source2");
// 即使傳入 .tar.gz也應該輸出 .tar
const outputTar = join(testDir, "output.tar.gz");
mkdirSync(sourceDir, { recursive: true });
writeFileSync(join(sourceDir, "file.txt"), "Content");
const result = await createTarArchive(sourceDir, outputTar);
// 結果應該是 .tar 而非 .tar.gz
expect(result.endsWith(".tar")).toBe(true);
expect(result).not.toContain(".tar.gz");
});
});
describe("End-to-End 測試", () => {
beforeEach(() => {
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true });
}
});
afterEach(() => {
if (existsSync(testDir)) {
rmSync(testDir, { recursive: true, force: true });
}
});
test("完整流程:上傳 -> 封裝 -> 下載資訊", async () => {
// 1. 模擬上傳
const uploadsDir = join(testDir, "uploads");
const outputDir = join(testDir, "output");
mkdirSync(uploadsDir, { recursive: true });
mkdirSync(outputDir, { recursive: true });
const uploadedFile = join(uploadsDir, "document.pdf");
writeFileSync(uploadedFile, "Fake PDF content");
// 2. 模擬轉換輸出
const conversionOutput = join(outputDir, "conversion_result");
mkdirSync(conversionOutput, { recursive: true });
writeFileSync(join(conversionOutput, "output.md"), "# Converted Document");
mkdirSync(join(conversionOutput, "images"), { recursive: true });
writeFileSync(join(conversionOutput, "images", "fig1.png"), "Fake image");
// 3. 建立 .tar 封裝
const tarPath = join(outputDir, "result.tar");
const archivePath = await createTarArchive(conversionOutput, tarPath);
expect(existsSync(archivePath)).toBe(true);
expect(archivePath.endsWith(".tar")).toBe(true);
// 4. 取得下載資訊
const downloadInfo = getChunkDownloadInfo(archivePath);
expect(downloadInfo).not.toBeNull();
expect(downloadInfo!.file_name).toBe("result.tar");
// 5. 驗證傳輸模式判斷
const fileStats = statSync(archivePath);
const shouldChunk = fileStats.size > CHUNK_THRESHOLD_BYTES;
expect(shouldUseChunkedDownload(archivePath)).toBe(shouldChunk);
});
});
describe("上傳會話管理測試", () => {
test("應正確建立和管理上傳會話", () => {
const uploadId = "test-upload-123";
const session = uploadSessionManager.createSession(
uploadId,
"user-1",
"job-1",
"large-file.pdf",
50 * 1024 * 1024, // 50MB
10, // 10 chunks
testDir
);
expect(session.upload_id).toBe(uploadId);
expect(session.total_chunks).toBe(10);
expect(session.received_chunks.size).toBe(0);
// 標記已接收 chunks
uploadSessionManager.markChunkReceived(uploadId, 0);
uploadSessionManager.markChunkReceived(uploadId, 1);
const updatedSession = uploadSessionManager.getSession(uploadId);
expect(updatedSession!.received_chunks.size).toBe(2);
expect(uploadSessionManager.isComplete(uploadId)).toBe(false);
// 清理
uploadSessionManager.removeSession(uploadId);
expect(uploadSessionManager.getSession(uploadId)).toBeUndefined();
});
});