feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小) - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器(支援直傳與 chunk) - downloadManager.ts: 後端下載管理器(支援直傳與 chunk) - archiveManager.ts: 封裝管理器(僅允許 .tar) - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk) - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB:直接傳輸 - 檔案 >10MB:使用 5MB chunks 分段傳輸 - 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip) - 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
|
|
@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process";
|
|||
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs";
|
||||
import { join, basename, dirname } from "node:path";
|
||||
import { ExecFileFn } from "./types";
|
||||
import { createConverterArchive, getArchiveFileName } from "../transfer";
|
||||
|
||||
export const properties = {
|
||||
from: {
|
||||
|
|
@ -14,19 +15,22 @@ export const properties = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Helper function to create a tar.gz archive from a directory
|
||||
* Helper function to create a .tar archive from a directory (no compression)
|
||||
*
|
||||
* ⚠️ 重要:僅使用 .tar 格式,禁止 .tar.gz / .tgz / .zip
|
||||
*/
|
||||
function createTarGzArchive(
|
||||
function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputTarGz: string,
|
||||
outputTar: string,
|
||||
execFile: ExecFileFn,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Use tar command to create gzipped archive
|
||||
// tar -czf <output.tar.gz> -C <sourceDir> .
|
||||
// Use tar command to create archive (without gzip compression)
|
||||
// tar -cf <output.tar> -C <sourceDir> .
|
||||
// 注意:使用 -cf 而非 -czf,避免 gzip 壓縮
|
||||
execFile(
|
||||
"tar",
|
||||
["-czf", outputTarGz, "-C", sourceDir, "."],
|
||||
["-cf", outputTar, "-C", sourceDir, "."],
|
||||
(error, stdout, stderr) => {
|
||||
if (error) {
|
||||
reject(`tar error: ${error}`);
|
||||
|
|
@ -122,15 +126,14 @@ export async function convert(
|
|||
// MinerU outputs to a subdirectory, find the actual output
|
||||
const mineruActualOutput = join(mineruOutputDir, "auto");
|
||||
|
||||
// Create tar.gz archive from the output directory
|
||||
const tarGzPath = targetPath.endsWith(".tar.gz")
|
||||
? targetPath
|
||||
: `${targetPath}.tar.gz`;
|
||||
// Create .tar archive from the output directory (不使用壓縮)
|
||||
// 強制使用 .tar 格式,禁止 .tar.gz
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
|
||||
// Ensure the parent directory exists
|
||||
const tarGzDir = dirname(tarGzPath);
|
||||
if (!existsSync(tarGzDir)) {
|
||||
mkdirSync(tarGzDir, { recursive: true });
|
||||
const tarDir = dirname(tarPath);
|
||||
if (!existsSync(tarDir)) {
|
||||
mkdirSync(tarDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Use the actual MinerU output directory for archiving
|
||||
|
|
@ -138,14 +141,14 @@ export async function convert(
|
|||
? mineruActualOutput
|
||||
: mineruOutputDir;
|
||||
|
||||
await createTarGzArchive(outputToArchive, tarGzPath, execFile);
|
||||
await createTarArchive(outputToArchive, tarPath, execFile);
|
||||
|
||||
// Clean up the temporary directory
|
||||
removeDir(mineruOutputDir);
|
||||
|
||||
resolve("Done");
|
||||
} catch (tarError) {
|
||||
reject(`Failed to create tar.gz archive: ${tarError}`);
|
||||
reject(`Failed to create .tar archive: ${tarError}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ import { convert } from "./pages/convert";
|
|||
import { deleteFile } from "./pages/deleteFile";
|
||||
import { deleteJob } from "./pages/deleteJob";
|
||||
import { download } from "./pages/download";
|
||||
import { downloadChunk } from "./pages/downloadChunk";
|
||||
import { history } from "./pages/history";
|
||||
import { listConverters } from "./pages/listConverters";
|
||||
import { results } from "./pages/results";
|
||||
import { root } from "./pages/root";
|
||||
import { upload } from "./pages/upload";
|
||||
import { uploadChunk, uploadInfo } from "./pages/uploadChunk";
|
||||
import { user } from "./pages/user";
|
||||
import { healthcheck } from "./pages/healthcheck";
|
||||
|
||||
|
|
@ -41,9 +43,12 @@ const app = new Elysia({
|
|||
.use(user)
|
||||
.use(root)
|
||||
.use(upload)
|
||||
.use(uploadChunk)
|
||||
.use(uploadInfo)
|
||||
.use(history)
|
||||
.use(convert)
|
||||
.use(download)
|
||||
.use(downloadChunk)
|
||||
.use(deleteJob)
|
||||
.use(results)
|
||||
.use(deleteFile)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import path from "node:path";
|
||||
import { Elysia } from "elysia";
|
||||
import sanitize from "sanitize-filename";
|
||||
import * as tar from "tar";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import { createJobArchive, shouldUseChunkedDownload } from "../transfer";
|
||||
|
||||
export const download = new Elysia()
|
||||
.use(userService)
|
||||
|
|
@ -45,18 +45,10 @@ export const download = new Elysia()
|
|||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const outputPath = `${outputDir}${userId}/${jobId}`;
|
||||
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
|
||||
|
||||
await tar.create(
|
||||
{
|
||||
file: outputTar,
|
||||
cwd: outputPath,
|
||||
filter: (path) => {
|
||||
return !path.match(".*\\.tar");
|
||||
},
|
||||
},
|
||||
["."],
|
||||
);
|
||||
|
||||
// 使用統一的封裝管理器建立 .tar(不壓縮)
|
||||
const outputTar = await createJobArchive(outputPath, jobId);
|
||||
|
||||
return Bun.file(outputTar);
|
||||
},
|
||||
{
|
||||
|
|
|
|||
178
src/pages/downloadChunk.tsx
Normal file
178
src/pages/downloadChunk.tsx
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Contents.CN Chunk 下載 API
|
||||
*
|
||||
* 處理大檔的分段下載
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { join } from "node:path";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
createChunkDownloadHeaders,
|
||||
getFileForDirectDownload,
|
||||
createDirectDownloadHeaders,
|
||||
CHUNK_SIZE_BYTES,
|
||||
} from "../transfer";
|
||||
import { statSync, existsSync } from "node:fs";
|
||||
|
||||
export const downloadChunk = new Elysia()
|
||||
.use(userService)
|
||||
/**
|
||||
* 取得檔案下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(filePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* 下載特定 chunk
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(filePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
if (!existsSync(archivePath)) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(archivePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(archivePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
);
|
||||
88
src/pages/uploadChunk.tsx
Normal file
88
src/pages/uploadChunk.tsx
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/**
|
||||
* Contents.CN Chunk 上傳 API
|
||||
*
|
||||
* 處理大檔的分段上傳
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { uploadsDir } from "../index";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
handleChunkUpload,
|
||||
shouldUseChunkedUpload,
|
||||
CHUNK_SIZE_BYTES,
|
||||
calculateChunkCount,
|
||||
} from "../transfer";
|
||||
|
||||
export const uploadChunk = new Elysia().use(userService).post(
|
||||
"/upload-chunk",
|
||||
async ({ body, user, cookie: { jobId } }) => {
|
||||
if (!jobId?.value) {
|
||||
return {
|
||||
success: false,
|
||||
message: "No active job session",
|
||||
};
|
||||
}
|
||||
|
||||
const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body;
|
||||
|
||||
const sanitizedFileName = sanitize(file_name);
|
||||
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
|
||||
|
||||
// 取得 chunk 資料
|
||||
const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk;
|
||||
|
||||
const result = await handleChunkUpload(
|
||||
upload_id,
|
||||
parseInt(chunk_index, 10),
|
||||
parseInt(total_chunks, 10),
|
||||
chunkData,
|
||||
sanitizedFileName,
|
||||
parseInt(total_size, 10),
|
||||
user.id,
|
||||
jobId.value,
|
||||
`${uploadsDir}${user.id}/`,
|
||||
userUploadsDir
|
||||
);
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
upload_id: t.String(),
|
||||
chunk_index: t.String(),
|
||||
total_chunks: t.String(),
|
||||
file_name: t.String(),
|
||||
total_size: t.String(),
|
||||
chunk: t.File(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* 取得上傳資訊(用於前端判斷是否需要 chunk 上傳)
|
||||
*/
|
||||
export const uploadInfo = new Elysia().use(userService).post(
|
||||
"/upload-info",
|
||||
async ({ body }) => {
|
||||
const { file_size } = body;
|
||||
const size = parseInt(file_size, 10);
|
||||
|
||||
const useChunked = shouldUseChunkedUpload(size);
|
||||
|
||||
return {
|
||||
use_chunked: useChunked,
|
||||
chunk_size: CHUNK_SIZE_BYTES,
|
||||
total_chunks: useChunked ? calculateChunkCount(size) : 1,
|
||||
};
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
file_size: t.String(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
154
src/transfer/archiveManager.ts
Normal file
154
src/transfer/archiveManager.ts
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
/**
|
||||
* Contents.CN 封裝管理器
|
||||
*
|
||||
* 統一處理多檔輸出的封裝:
|
||||
* - 唯一允許格式:.tar
|
||||
* - 禁止:.tar.gz, .tgz, .zip 等
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
|
||||
import { join, basename } from "node:path";
|
||||
import * as tar from "tar";
|
||||
import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants";
|
||||
|
||||
/**
|
||||
* 驗證封裝格式是否合法
|
||||
*/
|
||||
export function validateArchiveFormat(fileName: string): boolean {
|
||||
const lowerName = fileName.toLowerCase();
|
||||
|
||||
// 檢查是否使用禁止的格式
|
||||
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
|
||||
if (lowerName.endsWith(forbidden)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 檢查是否是允許的格式
|
||||
return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT);
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得正確的封裝檔名(強制使用 .tar)
|
||||
*/
|
||||
export function getArchiveFileName(baseName: string): string {
|
||||
const lowerName = baseName.toLowerCase();
|
||||
|
||||
// 移除任何禁止的副檔名
|
||||
let cleanName = baseName;
|
||||
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
|
||||
if (lowerName.endsWith(forbidden)) {
|
||||
cleanName = baseName.slice(0, -forbidden.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果已經是 .tar 結尾,直接返回
|
||||
if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) {
|
||||
return cleanName;
|
||||
}
|
||||
|
||||
// 否則加上 .tar
|
||||
return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立 .tar 封裝(不壓縮)
|
||||
*
|
||||
* @param sourceDir - 來源目錄
|
||||
* @param outputPath - 輸出的 .tar 檔案路徑
|
||||
* @param options - 額外選項
|
||||
*/
|
||||
export async function createTarArchive(
|
||||
sourceDir: string,
|
||||
outputPath: string,
|
||||
options: {
|
||||
filter?: (path: string) => boolean;
|
||||
prefix?: string;
|
||||
} = {}
|
||||
): Promise<string> {
|
||||
// 強制確保輸出是 .tar 格式
|
||||
const finalOutputPath = getArchiveFileName(outputPath);
|
||||
|
||||
// 確保輸出目錄存在
|
||||
const outputDir = join(finalOutputPath, "..");
|
||||
if (!existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// 取得要封裝的檔案列表
|
||||
const files = readdirSync(sourceDir);
|
||||
|
||||
// 預設過濾器:排除其他 tar 檔案
|
||||
const defaultFilter = (path: string) => !path.match(/\.tar$/i);
|
||||
const filter = options.filter || defaultFilter;
|
||||
|
||||
await tar.create(
|
||||
{
|
||||
file: finalOutputPath,
|
||||
cwd: sourceDir,
|
||||
filter: filter,
|
||||
// 不使用 gzip 壓縮
|
||||
gzip: false,
|
||||
},
|
||||
files.filter(f => filter(f))
|
||||
);
|
||||
|
||||
console.log(`Created tar archive: ${finalOutputPath}`);
|
||||
return finalOutputPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立用於多檔輸出的 .tar 封裝
|
||||
*
|
||||
* @param outputDir - 輸出目錄(包含多個轉換結果)
|
||||
* @param jobId - 任務 ID
|
||||
*/
|
||||
export async function createJobArchive(outputDir: string, jobId: string): Promise<string> {
|
||||
const archiveName = `converted_files_${jobId}.tar`;
|
||||
const archivePath = join(outputDir, archiveName);
|
||||
|
||||
await createTarArchive(outputDir, archivePath, {
|
||||
filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案
|
||||
});
|
||||
|
||||
return archivePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 從目錄封裝 MinerU 等多檔輸出引擎的結果
|
||||
*
|
||||
* @param mineruOutputDir - MinerU 輸出目錄
|
||||
* @param targetPath - 目標 .tar 路徑
|
||||
*/
|
||||
export async function createConverterArchive(
|
||||
converterOutputDir: string,
|
||||
targetPath: string
|
||||
): Promise<string> {
|
||||
// 確保輸出使用 .tar 格式
|
||||
const tarPath = getArchiveFileName(targetPath);
|
||||
|
||||
await createTarArchive(converterOutputDir, tarPath);
|
||||
|
||||
return tarPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得封裝檔案資訊
|
||||
*/
|
||||
export function getArchiveInfo(archivePath: string): {
|
||||
exists: boolean;
|
||||
size: number;
|
||||
fileName: string;
|
||||
} | null {
|
||||
if (!existsSync(archivePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(archivePath);
|
||||
return {
|
||||
exists: true,
|
||||
size: stats.size,
|
||||
fileName: basename(archivePath),
|
||||
};
|
||||
}
|
||||
38
src/transfer/constants.ts
Normal file
38
src/transfer/constants.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸常數
|
||||
*
|
||||
* 這些常數定義了整個系統的檔案傳輸規則
|
||||
*/
|
||||
|
||||
/**
|
||||
* 檔案大小門檻(10MB)
|
||||
* - 小於等於此值:直接傳輸
|
||||
* - 大於此值:使用 chunk 分段傳輸
|
||||
*/
|
||||
export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
|
||||
|
||||
/**
|
||||
* 每個 chunk 的大小(5MB)
|
||||
*/
|
||||
export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
/**
|
||||
* 上傳 session 過期時間(毫秒)
|
||||
* 預設 1 小時
|
||||
*/
|
||||
export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
|
||||
|
||||
/**
|
||||
* 暫存目錄名稱
|
||||
*/
|
||||
export const CHUNK_TEMP_DIR = "chunks_temp";
|
||||
|
||||
/**
|
||||
* 允許的封裝格式(僅 .tar)
|
||||
*/
|
||||
export const ALLOWED_ARCHIVE_FORMAT = ".tar";
|
||||
|
||||
/**
|
||||
* 禁止的封裝格式
|
||||
*/
|
||||
export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"];
|
||||
123
src/transfer/downloadManager.ts
Normal file
123
src/transfer/downloadManager.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
/**
|
||||
* Contents.CN 後端下載管理器
|
||||
*
|
||||
* 統一處理所有檔案下載,包含:
|
||||
* - 小檔(≤10MB):直接回傳
|
||||
* - 大檔(>10MB):chunk 分段下載
|
||||
*/
|
||||
|
||||
import { existsSync, statSync, createReadStream } from "node:fs";
|
||||
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants";
|
||||
import type { ChunkDownloadInfo } from "./types";
|
||||
import { getTransferMode } from "./types";
|
||||
import { basename } from "node:path";
|
||||
|
||||
/**
|
||||
* 判斷檔案是否需要使用 chunk 下載
|
||||
*/
|
||||
export function shouldUseChunkedDownload(filePath: string): boolean {
|
||||
if (!existsSync(filePath)) {
|
||||
return false;
|
||||
}
|
||||
const stats = statSync(filePath);
|
||||
return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked";
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得檔案的 chunk 下載資訊
|
||||
*/
|
||||
export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(filePath);
|
||||
const fileName = basename(filePath);
|
||||
|
||||
return {
|
||||
total_size: stats.size,
|
||||
total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES),
|
||||
chunk_size: CHUNK_SIZE_BYTES,
|
||||
file_name: fileName,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得特定 chunk 的資料
|
||||
*/
|
||||
export async function getChunk(
|
||||
filePath: string,
|
||||
chunkIndex: number,
|
||||
chunkSize: number = CHUNK_SIZE_BYTES
|
||||
): Promise<Buffer | null> {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stats = statSync(filePath);
|
||||
const start = chunkIndex * chunkSize;
|
||||
const end = Math.min(start + chunkSize, stats.size);
|
||||
|
||||
if (start >= stats.size) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
const stream = createReadStream(filePath, { start, end: end - 1 });
|
||||
|
||||
stream.on("data", (chunk) => {
|
||||
chunks.push(chunk as Buffer);
|
||||
});
|
||||
|
||||
stream.on("end", () => {
|
||||
resolve(Buffer.concat(chunks));
|
||||
});
|
||||
|
||||
stream.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接取得完整檔案(小檔用)
|
||||
*/
|
||||
export function getFileForDirectDownload(filePath: string): ReturnType<typeof Bun.file> | null {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
return Bun.file(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立 chunk 下載回應的 headers
|
||||
*/
|
||||
export function createChunkDownloadHeaders(
|
||||
info: ChunkDownloadInfo,
|
||||
chunkIndex: number,
|
||||
chunkData: Buffer
|
||||
): Record<string, string> {
|
||||
const start = chunkIndex * info.chunk_size;
|
||||
const end = start + chunkData.length - 1;
|
||||
|
||||
return {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": chunkData.length.toString(),
|
||||
"Content-Range": `bytes ${start}-${end}/${info.total_size}`,
|
||||
"Accept-Ranges": "bytes",
|
||||
"X-Chunk-Index": chunkIndex.toString(),
|
||||
"X-Total-Chunks": info.total_chunks.toString(),
|
||||
"X-File-Name": encodeURIComponent(info.file_name),
|
||||
"X-Total-Size": info.total_size.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立直接下載回應的 headers
|
||||
*/
|
||||
export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record<string, string> {
|
||||
return {
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Length": fileSize.toString(),
|
||||
"Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`,
|
||||
};
|
||||
}
|
||||
56
src/transfer/index.ts
Normal file
56
src/transfer/index.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* Contents.CN 全域檔案傳輸模組
|
||||
*
|
||||
* 統一匯出所有傳輸相關功能
|
||||
*/
|
||||
|
||||
// 常數
|
||||
export {
|
||||
CHUNK_THRESHOLD_BYTES,
|
||||
CHUNK_SIZE_BYTES,
|
||||
UPLOAD_SESSION_TIMEOUT_MS,
|
||||
CHUNK_TEMP_DIR,
|
||||
ALLOWED_ARCHIVE_FORMAT,
|
||||
FORBIDDEN_ARCHIVE_FORMATS,
|
||||
} from "./constants";
|
||||
|
||||
// 類型
|
||||
export type {
|
||||
ChunkUploadRequest,
|
||||
ChunkUploadResponse,
|
||||
DirectUploadResponse,
|
||||
ChunkDownloadInfo,
|
||||
UploadSession,
|
||||
TransferMode,
|
||||
} from "./types";
|
||||
|
||||
export { getTransferMode } from "./types";
|
||||
|
||||
// 上傳管理
|
||||
export {
|
||||
uploadSessionManager,
|
||||
shouldUseChunkedUpload,
|
||||
handleDirectUpload,
|
||||
handleChunkUpload,
|
||||
calculateChunkCount,
|
||||
} from "./uploadManager";
|
||||
|
||||
// 下載管理
|
||||
export {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
getFileForDirectDownload,
|
||||
createChunkDownloadHeaders,
|
||||
createDirectDownloadHeaders,
|
||||
} from "./downloadManager";
|
||||
|
||||
// 封裝管理
|
||||
export {
|
||||
validateArchiveFormat,
|
||||
getArchiveFileName,
|
||||
createTarArchive,
|
||||
createJobArchive,
|
||||
createConverterArchive,
|
||||
getArchiveInfo,
|
||||
} from "./archiveManager";
|
||||
94
src/transfer/types.ts
Normal file
94
src/transfer/types.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
/**
|
||||
* Contents.CN 檔案傳輸類型定義
|
||||
*/
|
||||
|
||||
/**
|
||||
* Chunk 上傳請求結構
|
||||
*/
|
||||
export interface ChunkUploadRequest {
|
||||
/** 上傳會話 ID(UUID) */
|
||||
upload_id: string;
|
||||
/** 當前 chunk 索引(從 0 開始) */
|
||||
chunk_index: number;
|
||||
/** 總 chunk 數量 */
|
||||
total_chunks: number;
|
||||
/** chunk 數據 */
|
||||
data: Blob | ArrayBuffer;
|
||||
/** 原始檔案名稱 */
|
||||
file_name: string;
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 上傳回應結構
|
||||
*/
|
||||
export interface ChunkUploadResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
/** 已接收的 chunks 索引列表 */
|
||||
received_chunks?: number[];
|
||||
/** 是否所有 chunks 都已接收完成 */
|
||||
completed?: boolean;
|
||||
/** 合併後的檔案路徑(僅完成時返回) */
|
||||
file_path?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 直接上傳回應結構
|
||||
*/
|
||||
export interface DirectUploadResponse {
|
||||
success: boolean;
|
||||
message: string;
|
||||
file_path?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk 下載資訊
|
||||
*/
|
||||
export interface ChunkDownloadInfo {
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
/** chunk 數量 */
|
||||
total_chunks: number;
|
||||
/** 每個 chunk 大小 */
|
||||
chunk_size: number;
|
||||
/** 檔案名稱 */
|
||||
file_name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 上傳會話狀態
|
||||
*/
|
||||
export interface UploadSession {
|
||||
/** 上傳 ID */
|
||||
upload_id: string;
|
||||
/** 使用者 ID */
|
||||
user_id: string;
|
||||
/** Job ID */
|
||||
job_id: string;
|
||||
/** 檔案名稱 */
|
||||
file_name: string;
|
||||
/** 檔案總大小 */
|
||||
total_size: number;
|
||||
/** 總 chunk 數 */
|
||||
total_chunks: number;
|
||||
/** 已接收的 chunks */
|
||||
received_chunks: Set<number>;
|
||||
/** 建立時間 */
|
||||
created_at: Date;
|
||||
/** 暫存目錄路徑 */
|
||||
temp_dir: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 傳輸模式
|
||||
*/
|
||||
export type TransferMode = "direct" | "chunked";
|
||||
|
||||
/**
|
||||
* 根據檔案大小決定傳輸模式
|
||||
*/
|
||||
export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode {
|
||||
return fileSize <= thresholdBytes ? "direct" : "chunked";
|
||||
}
|
||||
303
src/transfer/uploadManager.ts
Normal file
303
src/transfer/uploadManager.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
/**
|
||||
* Contents.CN 後端上傳管理器
|
||||
*
|
||||
* 統一處理所有檔案上傳,包含:
|
||||
* - 小檔(≤10MB):直接接收
|
||||
* - 大檔(>10MB):chunk 接收與合併
|
||||
*/
|
||||
|
||||
import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants";
|
||||
import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types";
|
||||
import { getTransferMode } from "./types";
|
||||
|
||||
/**
|
||||
* 上傳會話管理器
|
||||
* 用於追蹤進行中的 chunk 上傳
|
||||
*/
|
||||
class UploadSessionManager {
|
||||
private sessions: Map<string, UploadSession> = new Map();
|
||||
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor() {
|
||||
// 定期清理過期的 sessions
|
||||
this.cleanupInterval = setInterval(() => {
|
||||
this.cleanupExpiredSessions();
|
||||
}, 5 * 60 * 1000); // 每 5 分鐘清理一次
|
||||
}
|
||||
|
||||
/**
|
||||
* 建立新的上傳會話
|
||||
*/
|
||||
createSession(
|
||||
uploadId: string,
|
||||
userId: string,
|
||||
jobId: string,
|
||||
fileName: string,
|
||||
totalSize: number,
|
||||
totalChunks: number,
|
||||
baseTempDir: string
|
||||
): UploadSession {
|
||||
const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId);
|
||||
|
||||
if (!existsSync(tempDir)) {
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
}
|
||||
|
||||
const session: UploadSession = {
|
||||
upload_id: uploadId,
|
||||
user_id: userId,
|
||||
job_id: jobId,
|
||||
file_name: fileName,
|
||||
total_size: totalSize,
|
||||
total_chunks: totalChunks,
|
||||
received_chunks: new Set(),
|
||||
created_at: new Date(),
|
||||
temp_dir: tempDir,
|
||||
};
|
||||
|
||||
this.sessions.set(uploadId, session);
|
||||
return session;
|
||||
}
|
||||
|
||||
/**
|
||||
* 取得上傳會話
|
||||
*/
|
||||
getSession(uploadId: string): UploadSession | undefined {
|
||||
return this.sessions.get(uploadId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新已接收的 chunk
|
||||
*/
|
||||
markChunkReceived(uploadId: string, chunkIndex: number): boolean {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (!session) return false;
|
||||
|
||||
session.received_chunks.add(chunkIndex);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 檢查是否所有 chunks 都已接收
|
||||
*/
|
||||
isComplete(uploadId: string): boolean {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (!session) return false;
|
||||
|
||||
return session.received_chunks.size === session.total_chunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除會話並清理暫存檔案
|
||||
*/
|
||||
removeSession(uploadId: string): void {
|
||||
const session = this.sessions.get(uploadId);
|
||||
if (session && existsSync(session.temp_dir)) {
|
||||
rmSync(session.temp_dir, { recursive: true, force: true });
|
||||
}
|
||||
this.sessions.delete(uploadId);
|
||||
}
|
||||
|
||||
/**
|
||||
* 清理過期的會話
|
||||
*/
|
||||
private cleanupExpiredSessions(): void {
|
||||
const now = Date.now();
|
||||
for (const [uploadId, session] of this.sessions) {
|
||||
if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) {
|
||||
console.log(`Cleaning up expired upload session: ${uploadId}`);
|
||||
this.removeSession(uploadId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 停止清理計時器
|
||||
*/
|
||||
destroy(): void {
|
||||
if (this.cleanupInterval) {
|
||||
clearInterval(this.cleanupInterval);
|
||||
this.cleanupInterval = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 單例實例
|
||||
export const uploadSessionManager = new UploadSessionManager();
|
||||
|
||||
/**
|
||||
* 判斷檔案是否需要使用 chunk 上傳
|
||||
*/
|
||||
export function shouldUseChunkedUpload(fileSize: number): boolean {
|
||||
return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked";
|
||||
}
|
||||
|
||||
/**
|
||||
* 處理直接上傳(小檔)
|
||||
*/
|
||||
export async function handleDirectUpload(
|
||||
file: File,
|
||||
targetDir: string,
|
||||
fileName: string
|
||||
): Promise<DirectUploadResponse> {
|
||||
try {
|
||||
const targetPath = join(targetDir, fileName);
|
||||
|
||||
if (!existsSync(targetDir)) {
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
await Bun.write(targetPath, file);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "File uploaded successfully.",
|
||||
file_path: targetPath,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Direct upload failed:", error);
|
||||
return {
|
||||
success: false,
|
||||
message: `Upload failed: ${error}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 處理 chunk 上傳
|
||||
*/
|
||||
export async function handleChunkUpload(
|
||||
uploadId: string,
|
||||
chunkIndex: number,
|
||||
totalChunks: number,
|
||||
chunkData: ArrayBuffer | Blob | Buffer,
|
||||
fileName: string,
|
||||
totalSize: number,
|
||||
userId: string,
|
||||
jobId: string,
|
||||
baseTempDir: string,
|
||||
targetDir: string
|
||||
): Promise<ChunkUploadResponse> {
|
||||
try {
|
||||
// 取得或建立會話
|
||||
let session = uploadSessionManager.getSession(uploadId);
|
||||
|
||||
if (!session) {
|
||||
session = uploadSessionManager.createSession(
|
||||
uploadId,
|
||||
userId,
|
||||
jobId,
|
||||
fileName,
|
||||
totalSize,
|
||||
totalChunks,
|
||||
baseTempDir
|
||||
);
|
||||
}
|
||||
|
||||
// 驗證會話資訊一致性
|
||||
if (session.total_chunks !== totalChunks || session.file_name !== fileName) {
|
||||
return {
|
||||
success: false,
|
||||
message: "Session mismatch: inconsistent upload parameters",
|
||||
};
|
||||
}
|
||||
|
||||
// 儲存 chunk
|
||||
const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`);
|
||||
// 處理不同類型的 chunk 資料
|
||||
let data: ArrayBuffer | Buffer;
|
||||
if (chunkData instanceof Blob) {
|
||||
data = await chunkData.arrayBuffer();
|
||||
} else if (Buffer.isBuffer(chunkData)) {
|
||||
data = chunkData;
|
||||
} else {
|
||||
data = chunkData;
|
||||
}
|
||||
await Bun.write(chunkPath, data);
|
||||
|
||||
// 標記已接收
|
||||
uploadSessionManager.markChunkReceived(uploadId, chunkIndex);
|
||||
|
||||
// 檢查是否完成
|
||||
if (uploadSessionManager.isComplete(uploadId)) {
|
||||
// 合併所有 chunks
|
||||
const finalPath = await mergeChunks(session, targetDir);
|
||||
|
||||
// 清理會話
|
||||
uploadSessionManager.removeSession(uploadId);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: "Upload completed and merged successfully.",
|
||||
received_chunks: Array.from({ length: totalChunks }, (_, i) => i),
|
||||
completed: true,
|
||||
file_path: finalPath,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`,
|
||||
received_chunks: Array.from(session.received_chunks),
|
||||
completed: false,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Chunk upload failed:", error);
|
||||
return {
|
||||
success: false,
|
||||
message: `Chunk upload failed: ${error}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 合併所有 chunks 為完整檔案
|
||||
*/
|
||||
async function mergeChunks(session: UploadSession, targetDir: string): Promise<string> {
|
||||
const targetPath = join(targetDir, session.file_name);
|
||||
|
||||
if (!existsSync(targetDir)) {
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
// 讀取並排序所有 chunks
|
||||
const chunkFiles = readdirSync(session.temp_dir)
|
||||
.filter(f => f.startsWith("chunk_"))
|
||||
.sort();
|
||||
|
||||
// 建立輸出串流
|
||||
const writeStream = createWriteStream(targetPath);
|
||||
|
||||
// 依序寫入每個 chunk
|
||||
for (const chunkFile of chunkFiles) {
|
||||
const chunkPath = join(session.temp_dir, chunkFile);
|
||||
const chunkData = await Bun.file(chunkPath).arrayBuffer();
|
||||
writeStream.write(Buffer.from(chunkData));
|
||||
}
|
||||
|
||||
// 結束寫入
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writeStream.end((err: Error | null) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// 驗證檔案大小
|
||||
const finalStats = statSync(targetPath);
|
||||
if (finalStats.size !== session.total_size) {
|
||||
console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`);
|
||||
}
|
||||
|
||||
console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`);
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* 計算需要的 chunk 數量
|
||||
*/
|
||||
export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
|
||||
return Math.ceil(fileSize / chunkSize);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue