feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小) - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器(支援直傳與 chunk) - downloadManager.ts: 後端下載管理器(支援直傳與 chunk) - archiveManager.ts: 封裝管理器(僅允許 .tar) - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk) - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB:直接傳輸 - 檔案 >10MB:使用 5MB chunks 分段傳輸 - 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip) - 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
|
|
@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(mineruArgs).toContain("--table-mode");
|
||||
|
|
@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => {
|
|||
writeFileSync(join(autoDir, "output.md"), "# Test\n\n");
|
||||
callback(null, "MinerU conversion complete", "");
|
||||
} else if (cmd === "tar") {
|
||||
// Simulate tar.gz creation
|
||||
// Simulate .tar creation (no compression)
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(capturedArgs).toContain("--table-mode");
|
||||
|
|
@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("MinerU converter tar.gz output", () => {
|
||||
const testDir = "./test-output-mineru-targz";
|
||||
describe("MinerU converter .tar output (no compression)", () => {
|
||||
const testDir = "./test-output-mineru-tar";
|
||||
|
||||
beforeEach(() => {
|
||||
if (!existsSync(testDir)) {
|
||||
|
|
@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
});
|
||||
|
||||
test("should create tar.gz archive from output", async () => {
|
||||
test("should create .tar archive from output (without gzip)", async () => {
|
||||
let tarCalled = false;
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
|
|
@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
// 使用 .tar 格式
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
expect(result).toBe("Done");
|
||||
expect(tarCalled).toBe(true);
|
||||
expect(tarArgs).toContain("-czf");
|
||||
// 驗證使用 -cf 而非 -czf(不壓縮)
|
||||
expect(tarArgs).toContain("-cf");
|
||||
// 確保沒有使用 gzip 壓縮標誌
|
||||
expect(tarArgs).not.toContain("-czf");
|
||||
});
|
||||
|
||||
test("should reject on mineru error", async () => {
|
||||
|
|
@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
const targetPath = join(testDir, "output.tar");
|
||||
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
|
||||
.rejects.toMatch(/mineru error/);
|
||||
});
|
||||
|
||||
test("should use correct tar arguments for compression", async () => {
|
||||
test("should use correct tar arguments without compression", async () => {
|
||||
let tarArgs: string[] = [];
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
|
|
@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => {
|
|||
}
|
||||
};
|
||||
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz");
|
||||
// 使用 .tar 格式(不是 .tar.gz)
|
||||
const targetPath = join(testDir, "test_MINERU_md-t.tar");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// Verify tar is called with correct compression flags
|
||||
expect(tarArgs[0]).toBe("-czf");
|
||||
expect(tarArgs[1]).toContain(".tar.gz");
|
||||
// 驗證 tar 使用正確的無壓縮標誌
|
||||
expect(tarArgs[0]).toBe("-cf");
|
||||
expect(tarArgs[1]).toContain(".tar");
|
||||
expect(tarArgs[1]).not.toContain(".tar.gz");
|
||||
expect(tarArgs[2]).toBe("-C");
|
||||
expect(tarArgs[4]).toBe(".");
|
||||
});
|
||||
|
||||
test("should convert .tar.gz target path to .tar", async () => {
|
||||
let tarOutputPath = "";
|
||||
|
||||
const mockExecFile: ExecFileFn = (
|
||||
cmd: string,
|
||||
args: string[],
|
||||
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
|
||||
options?: any,
|
||||
) => {
|
||||
if (cmd === "mineru") {
|
||||
const outputDir = args[3];
|
||||
if (outputDir && !existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
const autoDir = join(outputDir, "auto");
|
||||
if (!existsSync(autoDir)) {
|
||||
mkdirSync(autoDir, { recursive: true });
|
||||
}
|
||||
writeFileSync(join(autoDir, "output.md"), "# Test");
|
||||
callback(null, "Done", "");
|
||||
} else if (cmd === "tar") {
|
||||
tarOutputPath = args[1]; // 輸出路徑
|
||||
callback(null, "Archive created", "");
|
||||
}
|
||||
};
|
||||
|
||||
// 即使傳入 .tar.gz,也應該被轉換為 .tar
|
||||
const targetPath = join(testDir, "output.tar.gz");
|
||||
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
|
||||
|
||||
// 驗證輸出是 .tar 而非 .tar.gz
|
||||
expect(tarOutputPath).toContain(".tar");
|
||||
expect(tarOutputPath).not.toContain(".tar.gz");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue