feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數（10MB 門檻、5MB chunk 大小） - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器（支援直傳與 chunk） - downloadManager.ts: 後端下載管理器（支援直傳與 chunk） - archiveManager.ts: 封裝管理器（僅允許 .tar） - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略（≤10MB 直傳，>10MB chunk） - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式（不壓縮），禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB：直接傳輸 - 檔案 >10MB：使用 5MB chunks 分段傳輸 - 多檔輸出：僅允許 .tar 封裝（禁止 .tar.gz/.zip） - 引擎層不感知 chunk（僅傳輸層處理）
This commit is contained in:
Your Name
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
|
|
@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式（不是 .tar.gz）
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(mineruArgs).toContain("--table-mode");
|
||||
|
|
@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式（不是 .tar.gz）
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(capturedArgs).toContain("--table-mode");
|
||||
|
|
@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("MinerU converter tar.gz output", () => {
|
||||
const testDir = "./test-output-mineru-targz";
|
||||
describe("MinerU converter .tar output (no compression)", () => {
|
||||
const testDir = "./test-output-mineru-tar";
|
||||
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
|
|
@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("should create tar.gz archive from output", async () => {
|
||||
test("should create .tar archive from output (without gzip)", async () => {
|
||||
let tarCalled = false;
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
|
|
@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(result).toBe("Done");
|
||||
expect(tarCalled).toBe(true);
|
||||
expect(tarArgs).toContain("-czf");
|
||||
// 驗證使用 -cf 而非 -czf（不壓縮）
|
||||
expect(tarArgs).toContain("-cf");
|
||||
// 確保沒有使用 gzip 壓縮標誌
|
||||
expect(tarArgs).not.toContain("-czf");
|
||||
});
|
||||
|
||||
test("should reject on mineru error", async () => {
|
||||
|
|
@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
|
||||
.rejects.toMatch(/mineru error/);
|
||||
});
|
||||
|
||||
test("should use correct tar arguments for compression", async () => {
|
||||
test("should use correct tar arguments without compression", async () => {
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
|
|
@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz");
|
||||
// 使用 .tar 格式（不是 .tar.gz）
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// Verify tar is called with correct compression flags
|
||||
expect(tarArgs[0]).toBe("-czf");
|
||||
expect(tarArgs[1]).toContain(".tar.gz");
|
||||
// 驗證 tar 使用正確的無壓縮標誌
|
||||
expect(tarArgs[0]).toBe("-cf");
|
||||
expect(tarArgs[1]).toContain(".tar");
|
||||
expect(tarArgs[1]).not.toContain(".tar.gz");
|
||||
expect(tarArgs[2]).toBe("-C");
|
||||
expect(tarArgs[4]).toBe(".");
|
||||
});
|
||||
|
||||
test("should convert .tar.gz target path to .tar", async () => {
|
||||
let tarOutputPath = "";
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
cmd: string,
|
||||
args: string[],
|
||||
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
|
||||
options?: any,
|
||||
) => {
|
||||
if (cmd === "mineru") {
|
||||
const outputDir = args[3];
|
||||
if (outputDir && !existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
const autoDir = join(outputDir, "auto");
|
||||
if (!existsSync(autoDir)) {
|
||||
mkdirSync(autoDir, { recursive: true });
|
||||
}
|
||||
writeFileSync(join(autoDir, "output.md"), "# Test");
|
||||
callback(null, "Done", "");
|
||||
} else if (cmd === "tar") {
|
||||
tarOutputPath = args[1]; // 輸出路徑
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
// 即使傳入 .tar.gz，也應該被轉換為 .tar
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// 驗證輸出是 .tar 而非 .tar.gz
|
||||
expect(tarOutputPath).toContain(".tar");
|
||||
expect(tarOutputPath).not.toContain(".tar.gz");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
208
tests/transfer/chunk-download.test.ts
Normal file
208
tests/transfer/chunk-download.test.ts
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
/**
|
||||
* Contents.CN Chunk 下載整合測試
|
||||
*
|
||||
* 測試大檔 chunk 下載的完整流程
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
createChunkDownloadHeaders,
|
||||
} from "../../src/transfer/downloadManager";
|
||||
import { CHUNK_SIZE_BYTES, CHUNK_THRESHOLD_BYTES } from "../../src/transfer/constants";
|
||||
|
||||
const testDir = "./test-output-chunk-download";
|
||||
|
||||
describe("Chunk 下載資訊測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("getChunkDownloadInfo 應返回正確的下載資訊", () => {
|
||||
const testFile = join(testDir, "large-file.bin");
|
||||
const fileSize = 25 * 1024 * 1024; // 25MB
|
||||
|
||||
// 建立一個假檔案（只寫入少量資料模擬）
|
||||
const content = Buffer.alloc(1024, "X"); // 1KB
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
|
||||
expect(info).not.toBeNull();
|
||||
expect(info!.file_name).toBe("large-file.bin");
|
||||
expect(info!.total_size).toBe(1024);
|
||||
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
|
||||
expect(info!.total_chunks).toBe(1); // 1KB 只需要 1 chunk
|
||||
});
|
||||
|
||||
test("不存在的檔案應返回 null", () => {
|
||||
const info = getChunkDownloadInfo(join(testDir, "non-existent.bin"));
|
||||
expect(info).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 讀取測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("getChunk 應正確讀取指定的 chunk", async () => {
|
||||
const testFile = join(testDir, "chunked-file.bin");
|
||||
|
||||
// 建立測試檔案：20 bytes，分成 4 個 5-byte chunks
|
||||
const content = "AAAAABBBBBCCCCCDDDD"; // 19 bytes
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
// 讀取第一個 chunk（5 bytes）
|
||||
const chunk0 = await getChunk(testFile, 0, 5);
|
||||
expect(chunk0!.toString()).toBe("AAAAA");
|
||||
|
||||
// 讀取第二個 chunk
|
||||
const chunk1 = await getChunk(testFile, 1, 5);
|
||||
expect(chunk1!.toString()).toBe("BBBBB");
|
||||
|
||||
// 讀取第三個 chunk
|
||||
const chunk2 = await getChunk(testFile, 2, 5);
|
||||
expect(chunk2!.toString()).toBe("CCCCC");
|
||||
|
||||
// 讀取最後一個 chunk（只有 4 bytes）
|
||||
const chunk3 = await getChunk(testFile, 3, 5);
|
||||
expect(chunk3!.toString()).toBe("DDDD");
|
||||
});
|
||||
|
||||
test("讀取超出範圍的 chunk 應返回 null", async () => {
|
||||
const testFile = join(testDir, "small-file.txt");
|
||||
writeFileSync(testFile, "Hello");
|
||||
|
||||
const chunk = await getChunk(testFile, 10, 5); // 超出範圍
|
||||
expect(chunk).toBeNull();
|
||||
});
|
||||
|
||||
test("不存在的檔案應返回 null", async () => {
|
||||
const chunk = await getChunk(join(testDir, "non-existent.bin"), 0, 5);
|
||||
expect(chunk).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 下載 Headers 測試", () => {
|
||||
test("createChunkDownloadHeaders 應生成正確的 headers", () => {
|
||||
const info = {
|
||||
total_size: 1000,
|
||||
total_chunks: 4,
|
||||
chunk_size: 256,
|
||||
file_name: "test-file.bin",
|
||||
};
|
||||
|
||||
const chunkData = Buffer.alloc(256, "X");
|
||||
const headers = createChunkDownloadHeaders(info, 0, chunkData);
|
||||
|
||||
expect(headers["Content-Type"]).toBe("application/octet-stream");
|
||||
expect(headers["Content-Length"]).toBe("256");
|
||||
expect(headers["Content-Range"]).toBe("bytes 0-255/1000");
|
||||
expect(headers["Accept-Ranges"]).toBe("bytes");
|
||||
expect(headers["X-Chunk-Index"]).toBe("0");
|
||||
expect(headers["X-Total-Chunks"]).toBe("4");
|
||||
expect(headers["X-Total-Size"]).toBe("1000");
|
||||
});
|
||||
|
||||
test("最後一個 chunk 的 headers 應正確", () => {
|
||||
const info = {
|
||||
total_size: 1000,
|
||||
total_chunks: 4,
|
||||
chunk_size: 256,
|
||||
file_name: "test-file.bin",
|
||||
};
|
||||
|
||||
// 最後一個 chunk 只有 232 bytes (1000 - 256*3)
|
||||
const chunkData = Buffer.alloc(232, "Z");
|
||||
const headers = createChunkDownloadHeaders(info, 3, chunkData);
|
||||
|
||||
expect(headers["Content-Length"]).toBe("232");
|
||||
expect(headers["Content-Range"]).toBe("bytes 768-999/1000");
|
||||
expect(headers["X-Chunk-Index"]).toBe("3");
|
||||
});
|
||||
});
|
||||
|
||||
describe("傳輸模式判斷測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔（≤10MB）不應使用 chunk 下載", () => {
|
||||
const smallFile = join(testDir, "small.txt");
|
||||
writeFileSync(smallFile, "Small file content");
|
||||
|
||||
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
|
||||
});
|
||||
|
||||
test("不存在的檔案不應使用 chunk 下載", () => {
|
||||
expect(shouldUseChunkedDownload(join(testDir, "non-existent.bin"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("完整 chunk 下載流程測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("應能完整讀取並合併所有 chunks", async () => {
|
||||
const testFile = join(testDir, "complete-file.bin");
|
||||
const originalContent = "AAAAAABBBBBBCCCCCCDDDDDDEEEEEE"; // 30 bytes
|
||||
writeFileSync(testFile, originalContent);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
expect(info).not.toBeNull();
|
||||
|
||||
// 模擬分段讀取（每個 chunk 10 bytes）
|
||||
const chunkSize = 10;
|
||||
const totalChunks = Math.ceil(originalContent.length / chunkSize);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < totalChunks; i++) {
|
||||
const chunk = await getChunk(testFile, i, chunkSize);
|
||||
if (chunk) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
// 合併所有 chunks
|
||||
const mergedContent = Buffer.concat(chunks).toString();
|
||||
expect(mergedContent).toBe(originalContent);
|
||||
});
|
||||
});
|
||||
178
tests/transfer/chunk-upload.test.ts
Normal file
178
tests/transfer/chunk-upload.test.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Contents.CN Chunk 上傳整合測試
|
||||
*
|
||||
* 測試大檔 chunk 上傳的完整流程
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
handleChunkUpload,
|
||||
uploadSessionManager,
|
||||
calculateChunkCount,
|
||||
} from "../../src/transfer/uploadManager";
|
||||
import { CHUNK_SIZE_BYTES } from "../../src/transfer/constants";
|
||||
|
||||
const testDir = "./test-output-chunk-upload";
|
||||
|
||||
describe("Chunk 上傳整合測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("多 chunk 上傳應正確合併", async () => {
|
||||
const uploadId = "test-chunked-upload-1";
|
||||
const fileName = "large-file.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
// 建立測試資料（模擬 15MB 檔案，分 3 個 chunks）
|
||||
const chunkSize = 5 * 1024; // 使用較小的 chunk 以便測試
|
||||
const chunk1 = Buffer.alloc(chunkSize, "A");
|
||||
const chunk2 = Buffer.alloc(chunkSize, "B");
|
||||
const chunk3 = Buffer.alloc(chunkSize, "C");
|
||||
const totalSize = chunkSize * 3;
|
||||
const totalChunks = 3;
|
||||
|
||||
const baseTempDir = join(testDir, "temp");
|
||||
const targetDir = join(testDir, "uploads");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 上傳 chunk 1
|
||||
const result1 = await handleChunkUpload(
|
||||
uploadId, 0, totalChunks, chunk1, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result1.success).toBe(true);
|
||||
expect(result1.completed).toBe(false);
|
||||
expect(result1.received_chunks).toContain(0);
|
||||
|
||||
// 上傳 chunk 2
|
||||
const result2 = await handleChunkUpload(
|
||||
uploadId, 1, totalChunks, chunk2, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result2.success).toBe(true);
|
||||
expect(result2.completed).toBe(false);
|
||||
|
||||
// 上傳 chunk 3（最後一個）
|
||||
const result3 = await handleChunkUpload(
|
||||
uploadId, 2, totalChunks, chunk3, fileName, totalSize,
|
||||
userId, jobId, baseTempDir, targetDir
|
||||
);
|
||||
expect(result3.success).toBe(true);
|
||||
expect(result3.completed).toBe(true);
|
||||
expect(result3.file_path).toBeDefined();
|
||||
|
||||
// 驗證合併後的檔案
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
expect(existsSync(mergedFile)).toBe(true);
|
||||
|
||||
const mergedContent = readFileSync(mergedFile);
|
||||
expect(mergedContent.length).toBe(totalSize);
|
||||
|
||||
// 驗證內容正確性
|
||||
const expectedContent = Buffer.concat([chunk1, chunk2, chunk3]);
|
||||
expect(mergedContent.equals(expectedContent)).toBe(true);
|
||||
});
|
||||
|
||||
test("亂序上傳 chunks 也應正確合併", async () => {
|
||||
const uploadId = "test-chunked-upload-2";
|
||||
const fileName = "out-of-order.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
const chunkSize = 1024;
|
||||
const chunk0 = Buffer.alloc(chunkSize, "0");
|
||||
const chunk1 = Buffer.alloc(chunkSize, "1");
|
||||
const chunk2 = Buffer.alloc(chunkSize, "2");
|
||||
const chunk3 = Buffer.alloc(chunkSize, "3");
|
||||
const totalSize = chunkSize * 4;
|
||||
const totalChunks = 4;
|
||||
|
||||
const baseTempDir = join(testDir, "temp2");
|
||||
const targetDir = join(testDir, "uploads2");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 亂序上傳：3, 1, 0, 2
|
||||
await handleChunkUpload(uploadId, 3, totalChunks, chunk3, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
const finalResult = await handleChunkUpload(uploadId, 2, totalChunks, chunk2, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
expect(finalResult.success).toBe(true);
|
||||
expect(finalResult.completed).toBe(true);
|
||||
|
||||
// 驗證檔案順序正確
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
const mergedContent = readFileSync(mergedFile);
|
||||
|
||||
// 即使亂序上傳，合併後應該按正確順序
|
||||
const expectedContent = Buffer.concat([chunk0, chunk1, chunk2, chunk3]);
|
||||
expect(mergedContent.equals(expectedContent)).toBe(true);
|
||||
});
|
||||
|
||||
test("重複上傳相同 chunk 應被處理", async () => {
|
||||
const uploadId = "test-chunked-upload-3";
|
||||
const fileName = "duplicate-chunk.bin";
|
||||
const userId = "test-user";
|
||||
const jobId = "test-job";
|
||||
|
||||
const chunkSize = 512;
|
||||
const chunk0 = Buffer.alloc(chunkSize, "X");
|
||||
const chunk1 = Buffer.alloc(chunkSize, "Y");
|
||||
const totalSize = chunkSize * 2;
|
||||
const totalChunks = 2;
|
||||
|
||||
const baseTempDir = join(testDir, "temp3");
|
||||
const targetDir = join(testDir, "uploads3");
|
||||
mkdirSync(baseTempDir, { recursive: true });
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
|
||||
// 上傳 chunk 0
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
// 重複上傳 chunk 0
|
||||
await handleChunkUpload(uploadId, 0, totalChunks, chunk0, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
// 上傳 chunk 1
|
||||
const result = await handleChunkUpload(uploadId, 1, totalChunks, chunk1, fileName, totalSize, userId, jobId, baseTempDir, targetDir);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.completed).toBe(true);
|
||||
|
||||
// 驗證檔案大小正確（不應該因為重複上傳而變大）
|
||||
const mergedFile = join(targetDir, fileName);
|
||||
const stats = statSync(mergedFile);
|
||||
expect(stats.size).toBe(totalSize);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 數量計算測試", () => {
|
||||
test("應正確計算不同大小檔案的 chunk 數量", () => {
|
||||
const chunkSize = CHUNK_SIZE_BYTES;
|
||||
|
||||
// 剛好 1 個 chunk
|
||||
expect(calculateChunkCount(chunkSize)).toBe(1);
|
||||
|
||||
// 多 1 byte 需要 2 個 chunks
|
||||
expect(calculateChunkCount(chunkSize + 1)).toBe(2);
|
||||
|
||||
// 50MB 需要 10 個 chunks
|
||||
expect(calculateChunkCount(50 * 1024 * 1024)).toBe(10);
|
||||
|
||||
// 1 byte 需要 1 個 chunk
|
||||
expect(calculateChunkCount(1)).toBe(1);
|
||||
});
|
||||
});
|
||||
331
tests/transfer/transfer.test.ts
Normal file
331
tests/transfer/transfer.test.ts
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸機制測試
|
||||
*
|
||||
* 測試項目：
|
||||
* 1. 小檔直傳測試（≤10MB）
|
||||
* 2. 大檔 chunk 上傳測試（>10MB）
|
||||
* 3. chunk 合併正確性測試
|
||||
* 4. .tar 封裝測試
|
||||
* 5. End-to-End 測試
|
||||
*/
|
||||
|
||||
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
|
||||
import { mkdirSync, existsSync, writeFileSync, rmSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
shouldUseChunkedUpload,
|
||||
calculateChunkCount,
|
||||
handleDirectUpload,
|
||||
handleChunkUpload,
|
||||
uploadSessionManager,
|
||||
} from "../../src/transfer/uploadManager";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
} from "../../src/transfer/downloadManager";
|
||||
import {
|
||||
validateArchiveFormat,
|
||||
getArchiveFileName,
|
||||
createTarArchive,
|
||||
} from "../../src/transfer/archiveManager";
|
||||
import {
|
||||
CHUNK_THRESHOLD_BYTES,
|
||||
CHUNK_SIZE_BYTES,
|
||||
ALLOWED_ARCHIVE_FORMAT,
|
||||
FORBIDDEN_ARCHIVE_FORMATS,
|
||||
} from "../../src/transfer/constants";
|
||||
import { getTransferMode } from "../../src/transfer/types";
|
||||
|
||||
const testDir = "./test-output-transfer";
|
||||
|
||||
describe("傳輸常數測試", () => {
|
||||
test("CHUNK_THRESHOLD_BYTES 應為 10MB", () => {
|
||||
expect(CHUNK_THRESHOLD_BYTES).toBe(10 * 1024 * 1024);
|
||||
});
|
||||
|
||||
test("CHUNK_SIZE_BYTES 應為 5MB", () => {
|
||||
expect(CHUNK_SIZE_BYTES).toBe(5 * 1024 * 1024);
|
||||
});
|
||||
|
||||
test("唯一允許的封裝格式應為 .tar", () => {
|
||||
expect(ALLOWED_ARCHIVE_FORMAT).toBe(".tar");
|
||||
});
|
||||
|
||||
test("禁止的封裝格式應包含 .tar.gz, .tgz, .zip, .gz", () => {
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tar.gz");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".tgz");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".zip");
|
||||
expect(FORBIDDEN_ARCHIVE_FORMATS).toContain(".gz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("傳輸模式判斷測試", () => {
|
||||
test("≤10MB 應使用直傳模式", () => {
|
||||
expect(getTransferMode(10 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(5 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(1024, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
expect(getTransferMode(0, CHUNK_THRESHOLD_BYTES)).toBe("direct");
|
||||
});
|
||||
|
||||
test(">10MB 應使用 chunk 模式", () => {
|
||||
expect(getTransferMode(10 * 1024 * 1024 + 1, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
expect(getTransferMode(50 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
expect(getTransferMode(100 * 1024 * 1024, CHUNK_THRESHOLD_BYTES)).toBe("chunked");
|
||||
});
|
||||
|
||||
test("shouldUseChunkedUpload 函數應正確判斷", () => {
|
||||
expect(shouldUseChunkedUpload(10 * 1024 * 1024)).toBe(false);
|
||||
expect(shouldUseChunkedUpload(10 * 1024 * 1024 + 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Chunk 數量計算測試", () => {
|
||||
test("計算 chunk 數量應正確", () => {
|
||||
// 10MB / 5MB = 2 chunks
|
||||
expect(calculateChunkCount(10 * 1024 * 1024)).toBe(2);
|
||||
|
||||
// 15MB / 5MB = 3 chunks
|
||||
expect(calculateChunkCount(15 * 1024 * 1024)).toBe(3);
|
||||
|
||||
// 1 byte 也需要 1 chunk
|
||||
expect(calculateChunkCount(1)).toBe(1);
|
||||
|
||||
// 5MB + 1 byte = 2 chunks
|
||||
expect(calculateChunkCount(5 * 1024 * 1024 + 1)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("封裝格式驗證測試", () => {
|
||||
test("validateArchiveFormat 應正確驗證格式", () => {
|
||||
// 允許的格式
|
||||
expect(validateArchiveFormat("output.tar")).toBe(true);
|
||||
expect(validateArchiveFormat("my_archive.tar")).toBe(true);
|
||||
|
||||
// 禁止的格式
|
||||
expect(validateArchiveFormat("output.tar.gz")).toBe(false);
|
||||
expect(validateArchiveFormat("output.tgz")).toBe(false);
|
||||
expect(validateArchiveFormat("output.zip")).toBe(false);
|
||||
expect(validateArchiveFormat("output.gz")).toBe(false);
|
||||
});
|
||||
|
||||
test("getArchiveFileName 應強制轉換為 .tar", () => {
|
||||
// 已經是 .tar
|
||||
expect(getArchiveFileName("output.tar")).toBe("output.tar");
|
||||
|
||||
// 轉換 .tar.gz -> .tar
|
||||
expect(getArchiveFileName("output.tar.gz")).toBe("output.tar");
|
||||
|
||||
// 轉換 .tgz -> .tar
|
||||
expect(getArchiveFileName("output.tgz")).toBe("output.tar");
|
||||
|
||||
// 無副檔名 -> 加上 .tar
|
||||
expect(getArchiveFileName("output")).toBe("output.tar");
|
||||
|
||||
// .zip 也是禁止的格式，會被移除並加上 .tar
|
||||
expect(getArchiveFileName("output.zip")).toBe("output.tar");
|
||||
|
||||
// .gz 也是禁止的格式
|
||||
expect(getArchiveFileName("output.gz")).toBe("output.tar");
|
||||
});
|
||||
});
|
||||
|
||||
describe("上傳管理測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔直傳應成功", async () => {
|
||||
const targetDir = join(testDir, "uploads");
|
||||
const testContent = "Hello, World!";
|
||||
const file = new File([testContent], "test.txt", { type: "text/plain" });
|
||||
|
||||
const result = await handleDirectUpload(file as any, targetDir, "test.txt");
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(existsSync(join(targetDir, "test.txt"))).toBe(true);
|
||||
expect(readFileSync(join(targetDir, "test.txt"), "utf-8")).toBe(testContent);
|
||||
});
|
||||
});
|
||||
|
||||
describe("下載管理測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("小檔不應使用 chunk 下載", () => {
|
||||
const smallFile = join(testDir, "small.txt");
|
||||
writeFileSync(smallFile, "Small content");
|
||||
|
||||
expect(shouldUseChunkedDownload(smallFile)).toBe(false);
|
||||
});
|
||||
|
||||
test("getChunkDownloadInfo 應返回正確資訊", () => {
|
||||
const testFile = join(testDir, "test.txt");
|
||||
const content = "Test content for download";
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
const info = getChunkDownloadInfo(testFile);
|
||||
|
||||
expect(info).not.toBeNull();
|
||||
expect(info!.file_name).toBe("test.txt");
|
||||
expect(info!.total_size).toBe(content.length);
|
||||
expect(info!.chunk_size).toBe(CHUNK_SIZE_BYTES);
|
||||
});
|
||||
|
||||
test("getChunk 應正確讀取 chunk 資料", async () => {
|
||||
const testFile = join(testDir, "test.txt");
|
||||
const content = "ABCDEFGHIJ"; // 10 bytes
|
||||
writeFileSync(testFile, content);
|
||||
|
||||
// 讀取第一個 chunk（整個檔案，因為小於 chunk size）
|
||||
const chunk = await getChunk(testFile, 0, 5);
|
||||
|
||||
expect(chunk).not.toBeNull();
|
||||
expect(chunk!.toString()).toBe("ABCDE");
|
||||
|
||||
// 讀取第二個 chunk
|
||||
const chunk2 = await getChunk(testFile, 1, 5);
|
||||
expect(chunk2!.toString()).toBe("FGHIJ");
|
||||
});
|
||||
});
|
||||
|
||||
describe(".tar 封裝測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("createTarArchive 應建立 .tar 檔案", async () => {
|
||||
const sourceDir = join(testDir, "source");
|
||||
const outputTar = join(testDir, "output.tar");
|
||||
|
||||
mkdirSync(sourceDir, { recursive: true });
|
||||
writeFileSync(join(sourceDir, "file1.txt"), "Content 1");
|
||||
writeFileSync(join(sourceDir, "file2.txt"), "Content 2");
|
||||
|
||||
const result = await createTarArchive(sourceDir, outputTar);
|
||||
|
||||
expect(existsSync(result)).toBe(true);
|
||||
expect(result.endsWith(".tar")).toBe(true);
|
||||
expect(result).not.toContain(".tar.gz");
|
||||
});
|
||||
|
||||
test("createTarArchive 應強制使用 .tar 格式", async () => {
|
||||
const sourceDir = join(testDir, "source2");
|
||||
// 即使傳入 .tar.gz，也應該輸出 .tar
|
||||
const outputTar = join(testDir, "output.tar.gz");
|
||||
|
||||
mkdirSync(sourceDir, { recursive: true });
|
||||
writeFileSync(join(sourceDir, "file.txt"), "Content");
|
||||
|
||||
const result = await createTarArchive(sourceDir, outputTar);
|
||||
|
||||
// 結果應該是 .tar 而非 .tar.gz
|
||||
expect(result.endsWith(".tar")).toBe(true);
|
||||
expect(result).not.toContain(".tar.gz");
|
||||
});
|
||||
});
|
||||
|
||||
describe("End-to-End 測試", () => {
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(testDir)) {
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("完整流程：上傳 -> 封裝 -> 下載資訊", async () => {
|
||||
// 1. 模擬上傳
|
||||
const uploadsDir = join(testDir, "uploads");
|
||||
const outputDir = join(testDir, "output");
|
||||
mkdirSync(uploadsDir, { recursive: true });
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const uploadedFile = join(uploadsDir, "document.pdf");
|
||||
writeFileSync(uploadedFile, "Fake PDF content");
|
||||
|
||||
// 2. 模擬轉換輸出
|
||||
const conversionOutput = join(outputDir, "conversion_result");
|
||||
mkdirSync(conversionOutput, { recursive: true });
|
||||
writeFileSync(join(conversionOutput, "output.md"), "# Converted Document");
|
||||
mkdirSync(join(conversionOutput, "images"), { recursive: true });
|
||||
writeFileSync(join(conversionOutput, "images", "fig1.png"), "Fake image");
|
||||
|
||||
// 3. 建立 .tar 封裝
|
||||
const tarPath = join(outputDir, "result.tar");
|
||||
const archivePath = await createTarArchive(conversionOutput, tarPath);
|
||||
|
||||
expect(existsSync(archivePath)).toBe(true);
|
||||
expect(archivePath.endsWith(".tar")).toBe(true);
|
||||
|
||||
// 4. 取得下載資訊
|
||||
const downloadInfo = getChunkDownloadInfo(archivePath);
|
||||
expect(downloadInfo).not.toBeNull();
|
||||
expect(downloadInfo!.file_name).toBe("result.tar");
|
||||
|
||||
// 5. 驗證傳輸模式判斷
|
||||
const fileStats = statSync(archivePath);
|
||||
const shouldChunk = fileStats.size > CHUNK_THRESHOLD_BYTES;
|
||||
expect(shouldUseChunkedDownload(archivePath)).toBe(shouldChunk);
|
||||
});
|
||||
});
|
||||
|
||||
describe("上傳會話管理測試", () => {
|
||||
test("應正確建立和管理上傳會話", () => {
|
||||
const uploadId = "test-upload-123";
|
||||
const session = uploadSessionManager.createSession(
|
||||
uploadId,
|
||||
"user-1",
|
||||
"job-1",
|
||||
"large-file.pdf",
|
||||
50 * 1024 * 1024, // 50MB
|
||||
10, // 10 chunks
|
||||
testDir
|
||||
);
|
||||
|
||||
expect(session.upload_id).toBe(uploadId);
|
||||
expect(session.total_chunks).toBe(10);
|
||||
expect(session.received_chunks.size).toBe(0);
|
||||
|
||||
// 標記已接收 chunks
|
||||
uploadSessionManager.markChunkReceived(uploadId, 0);
|
||||
uploadSessionManager.markChunkReceived(uploadId, 1);
|
||||
|
||||
const updatedSession = uploadSessionManager.getSession(uploadId);
|
||||
expect(updatedSession!.received_chunks.size).toBe(2);
|
||||
expect(uploadSessionManager.isComplete(uploadId)).toBe(false);
|
||||
|
||||
// 清理
|
||||
uploadSessionManager.removeSession(uploadId);
|
||||
expect(uploadSessionManager.getSession(uploadId)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue