feat: 實作全域檔案傳輸機制與 .tar 封裝規範

## 主要變更

### 新增全域檔案傳輸模組 (src/transfer/)
- constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小)
- types.ts: 傳輸類型定義
- uploadManager.ts: 後端上傳管理器(支援直傳與 chunk)
- downloadManager.ts: 後端下載管理器(支援直傳與 chunk)
- archiveManager.ts: 封裝管理器(僅允許 .tar)
- index.ts: 統一匯出

### 新增 API 端點
- uploadChunk.tsx: Chunk 上傳 API
- downloadChunk.tsx: Chunk 下載 API

### 前端更新
- public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk)
- public/transfer.js: 前端傳輸管理模組

### 轉換器更新
- mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz
- download.tsx: 使用統一的封裝管理器

### 測試
- tests/transfer/: 完整傳輸機制測試套件
- tests/converters/mineru.test.ts: 更新以符合 .tar 規範

### 文件
- README.md: 新增檔案傳輸機制說明

## 設計原則
- 檔案 ≤10MB:直接傳輸
- 檔案 >10MB:使用 5MB chunks 分段傳輸
- 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip)
- 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
Your Name 2026-01-21 13:58:01 +08:00
parent f7ebc084ea
commit 5322f85721
18 changed files with 2387 additions and 61 deletions

View file

@ -67,12 +67,13 @@ describe("MinerU converter md-t mode", () => {
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
callback(null, "MinerU conversion complete", "");
} else if (cmd === "tar") {
// Simulate tar.gz creation
// Simulate .tar creation (no compression)
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "output.tar.gz");
// 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "output.tar");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(mineruArgs).toContain("--table-mode");
@ -118,12 +119,13 @@ describe("MinerU converter md-i mode", () => {
writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)");
callback(null, "MinerU conversion complete", "");
} else if (cmd === "tar") {
// Simulate tar.gz creation
// Simulate .tar creation (no compression)
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "output.tar.gz");
// 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "output.tar");
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
expect(capturedArgs).toContain("--table-mode");
@ -131,8 +133,8 @@ describe("MinerU converter md-i mode", () => {
});
});
describe("MinerU converter tar.gz output", () => {
const testDir = "./test-output-mineru-targz";
describe("MinerU converter .tar output (no compression)", () => {
const testDir = "./test-output-mineru-tar";
beforeEach(() => {
if (!existsSync(testDir)) {
@ -146,7 +148,7 @@ describe("MinerU converter tar.gz output", () => {
}
});
test("should create tar.gz archive from output", async () => {
test("should create .tar archive from output (without gzip)", async () => {
let tarCalled = false;
let tarArgs: string[] = [];
@ -177,12 +179,16 @@ describe("MinerU converter tar.gz output", () => {
}
};
const targetPath = join(testDir, "output.tar.gz");
// 使用 .tar 格式
const targetPath = join(testDir, "output.tar");
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(result).toBe("Done");
expect(tarCalled).toBe(true);
expect(tarArgs).toContain("-czf");
// 驗證使用 -cf 而非 -czf不壓縮
expect(tarArgs).toContain("-cf");
// 確保沒有使用 gzip 壓縮標誌
expect(tarArgs).not.toContain("-czf");
});
test("should reject on mineru error", async () => {
@ -197,12 +203,12 @@ describe("MinerU converter tar.gz output", () => {
}
};
const targetPath = join(testDir, "output.tar.gz");
const targetPath = join(testDir, "output.tar");
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
.rejects.toMatch(/mineru error/);
});
test("should use correct tar arguments for compression", async () => {
test("should use correct tar arguments without compression", async () => {
let tarArgs: string[] = [];
const mockExecFile: ExecFileFn = (
@ -228,13 +234,50 @@ describe("MinerU converter tar.gz output", () => {
}
};
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz");
// 使用 .tar 格式(不是 .tar.gz
const targetPath = join(testDir, "test_MINERU_md-t.tar");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
// Verify tar is called with correct compression flags
expect(tarArgs[0]).toBe("-czf");
expect(tarArgs[1]).toContain(".tar.gz");
// 驗證 tar 使用正確的無壓縮標誌
expect(tarArgs[0]).toBe("-cf");
expect(tarArgs[1]).toContain(".tar");
expect(tarArgs[1]).not.toContain(".tar.gz");
expect(tarArgs[2]).toBe("-C");
expect(tarArgs[4]).toBe(".");
});
test("should convert .tar.gz target path to .tar", async () => {
let tarOutputPath = "";
const mockExecFile: ExecFileFn = (
cmd: string,
args: string[],
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "mineru") {
const outputDir = args[3];
if (outputDir && !existsSync(outputDir)) {
mkdirSync(outputDir, { recursive: true });
}
const autoDir = join(outputDir, "auto");
if (!existsSync(autoDir)) {
mkdirSync(autoDir, { recursive: true });
}
writeFileSync(join(autoDir, "output.md"), "# Test");
callback(null, "Done", "");
} else if (cmd === "tar") {
tarOutputPath = args[1]; // 輸出路徑
callback(null, "Archive created", "");
}
};
// 即使傳入 .tar.gz也應該被轉換為 .tar
const targetPath = join(testDir, "output.tar.gz");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
// 驗證輸出是 .tar 而非 .tar.gz
expect(tarOutputPath).toContain(".tar");
expect(tarOutputPath).not.toContain(".tar.gz");
});
});