feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小) - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器(支援直傳與 chunk) - downloadManager.ts: 後端下載管理器(支援直傳與 chunk) - archiveManager.ts: 封裝管理器(僅允許 .tar) - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk) - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB:直接傳輸 - 檔案 >10MB:使用 5MB chunks 分段傳輸 - 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip) - 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
|
|
@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process";
|
|||
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs";
|
||||
import { join, basename, dirname } from "node:path";
|
||||
import { ExecFileFn } from "./types";
|
||||
import { createConverterArchive, getArchiveFileName } from "../transfer";
|
||||
|
||||
export const properties = {
|
||||
from: {
|
||||
|
|
@ -14,19 +15,22 @@ export const properties = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Helper function to create a tar.gz archive from a directory
|
||||
* Helper function to create a .tar archive from a directory (no compression)
|
||||
*
|
||||
* ⚠️ 重要:僅使用 .tar 格式,禁止 .tar.gz / .tgz / .zip
|
||||
*/
|
||||
function createTarGzArchive(
|
||||
function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputTarGz: string,
|
||||
outputTar: string,
|
||||
execFile: ExecFileFn,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use tar command to create gzipped archive
|
||||
// tar -czf <output.tar.gz> -C <sourceDir> .
|
||||
// Use tar command to create archive (without gzip compression)
|
||||
// tar -cf <output.tar> -C <sourceDir> .
|
||||
// 注意:使用 -cf 而非 -czf,避免 gzip 壓縮
|
||||
execFile(
|
||||
"tar",
|
||||
["-czf", outputTarGz, "-C", sourceDir, "."],
|
||||
["-cf", outputTar, "-C", sourceDir, "."],
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`tar error: ${error}`);
|
||||
|
|
@ -122,15 +126,14 @@ export async function convert(
|
|||
// MinerU outputs to a subdirectory, find the actual output
|
||||
const mineruActualOutput = join(mineruOutputDir, "auto");
|
||||
|
||||
// Create tar.gz archive from the output directory
|
||||
const tarGzPath = targetPath.endsWith(".tar.gz")
|
||||
? targetPath
|
||||
: `${targetPath}.tar.gz`;
|
||||
// Create .tar archive from the output directory (不使用壓縮)
|
||||
// 強制使用 .tar 格式,禁止 .tar.gz
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
|
||||
// Ensure the parent directory exists
|
||||
const tarGzDir = dirname(tarGzPath);
|
||||
if (!existsSync(tarGzDir)) {
|
||||
mkdirSync(tarGzDir, { recursive: true });
|
||||
const tarDir = dirname(tarPath);
|
||||
if (!existsSync(tarDir)) {
|
||||
mkdirSync(tarDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Use the actual MinerU output directory for archiving
|
||||
|
|
@ -138,14 +141,14 @@ export async function convert(
|
|||
? mineruActualOutput
|
||||
: mineruOutputDir;
|
||||
|
||||
await createTarGzArchive(outputToArchive, tarGzPath, execFile);
|
||||
await createTarArchive(outputToArchive, tarPath, execFile);
|
||||
|
||||
// Clean up the temporary directory
|
||||
removeDir(mineruOutputDir);
|
||||
|
||||
resolve("Done");
|
||||
} catch (tarError) {
|
||||
reject(`Failed to create tar.gz archive: ${tarError}`);
|
||||
reject(`Failed to create .tar archive: ${tarError}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue