feat: 實作全域檔案傳輸機制與 .tar 封裝規範

## 主要變更

### 新增全域檔案傳輸模組 (src/transfer/)
- constants.ts: 定義傳輸常數(10MB 門檻、5MB chunk 大小)
- types.ts: 傳輸類型定義
- uploadManager.ts: 後端上傳管理器(支援直傳與 chunk)
- downloadManager.ts: 後端下載管理器(支援直傳與 chunk)
- archiveManager.ts: 封裝管理器(僅允許 .tar)
- index.ts: 統一匯出

### 新增 API 端點
- uploadChunk.tsx: Chunk 上傳 API
- downloadChunk.tsx: Chunk 下載 API

### 前端更新
- public/script.js: 整合智慧傳輸策略(≤10MB 直傳,>10MB chunk)
- public/transfer.js: 前端傳輸管理模組

### 轉換器更新
- mineru.ts: 改用 .tar 格式(不壓縮),禁止 .tar.gz
- download.tsx: 使用統一的封裝管理器

### 測試
- tests/transfer/: 完整傳輸機制測試套件
- tests/converters/mineru.test.ts: 更新以符合 .tar 規範

### 文件
- README.md: 新增檔案傳輸機制說明

## 設計原則
- 檔案 ≤10MB:直接傳輸
- 檔案 >10MB:使用 5MB chunks 分段傳輸
- 多檔輸出:僅允許 .tar 封裝(禁止 .tar.gz/.zip)
- 引擎層不感知 chunk(僅傳輸層處理)
This commit is contained in:
Your Name 2026-01-21 13:58:01 +08:00
parent f7ebc084ea
commit 5322f85721
18 changed files with 2387 additions and 61 deletions

View file

@ -2,6 +2,7 @@ import { execFile as execFileOriginal } from "node:child_process";
import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs";
import { join, basename, dirname } from "node:path";
import { ExecFileFn } from "./types";
import { createConverterArchive, getArchiveFileName } from "../transfer";
export const properties = {
from: {
@ -14,19 +15,22 @@ export const properties = {
};
/**
* Helper function to create a tar.gz archive from a directory
* Helper function to create a .tar archive from a directory (no compression)
*
* 使 .tar .tar.gz / .tgz / .zip
*/
function createTarGzArchive(
function createTarArchive(
sourceDir: string,
outputTarGz: string,
outputTar: string,
execFile: ExecFileFn,
): Promise<void> {
return new Promise((resolve, reject) => {
// Use tar command to create gzipped archive
// tar -czf <output.tar.gz> -C <sourceDir> .
// Use tar command to create archive (without gzip compression)
// tar -cf <output.tar> -C <sourceDir> .
// 注意:使用 -cf 而非 -czf避免 gzip 壓縮
execFile(
"tar",
["-czf", outputTarGz, "-C", sourceDir, "."],
["-cf", outputTar, "-C", sourceDir, "."],
(error, stdout, stderr) => {
if (error) {
reject(`tar error: ${error}`);
@ -122,15 +126,14 @@ export async function convert(
// MinerU outputs to a subdirectory, find the actual output
const mineruActualOutput = join(mineruOutputDir, "auto");
// Create tar.gz archive from the output directory
const tarGzPath = targetPath.endsWith(".tar.gz")
? targetPath
: `${targetPath}.tar.gz`;
// Create .tar archive from the output directory (不使用壓縮)
// 強制使用 .tar 格式,禁止 .tar.gz
const tarPath = getArchiveFileName(targetPath);
// Ensure the parent directory exists
const tarGzDir = dirname(tarGzPath);
if (!existsSync(tarGzDir)) {
mkdirSync(tarGzDir, { recursive: true });
const tarDir = dirname(tarPath);
if (!existsSync(tarDir)) {
mkdirSync(tarDir, { recursive: true });
}
// Use the actual MinerU output directory for archiving
@ -138,14 +141,14 @@ export async function convert(
? mineruActualOutput
: mineruOutputDir;
await createTarGzArchive(outputToArchive, tarGzPath, execFile);
await createTarArchive(outputToArchive, tarPath, execFile);
// Clean up the temporary directory
removeDir(mineruOutputDir);
resolve("Done");
} catch (tarError) {
reject(`Failed to create tar.gz archive: ${tarError}`);
reject(`Failed to create .tar archive: ${tarError}`);
}
});
});

View file

@ -11,11 +11,13 @@ import { convert } from "./pages/convert";
import { deleteFile } from "./pages/deleteFile";
import { deleteJob } from "./pages/deleteJob";
import { download } from "./pages/download";
import { downloadChunk } from "./pages/downloadChunk";
import { history } from "./pages/history";
import { listConverters } from "./pages/listConverters";
import { results } from "./pages/results";
import { root } from "./pages/root";
import { upload } from "./pages/upload";
import { uploadChunk, uploadInfo } from "./pages/uploadChunk";
import { user } from "./pages/user";
import { healthcheck } from "./pages/healthcheck";
@ -41,9 +43,12 @@ const app = new Elysia({
.use(user)
.use(root)
.use(upload)
.use(uploadChunk)
.use(uploadInfo)
.use(history)
.use(convert)
.use(download)
.use(downloadChunk)
.use(deleteJob)
.use(results)
.use(deleteFile)

View file

@ -1,11 +1,11 @@
import path from "node:path";
import { Elysia } from "elysia";
import sanitize from "sanitize-filename";
import * as tar from "tar";
import { outputDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { userService } from "./user";
import { createJobArchive, shouldUseChunkedDownload } from "../transfer";
export const download = new Elysia()
.use(userService)
@ -45,18 +45,10 @@ export const download = new Elysia()
const jobId = decodeURIComponent(params.jobId);
const outputPath = `${outputDir}${userId}/${jobId}`;
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
await tar.create(
{
file: outputTar,
cwd: outputPath,
filter: (path) => {
return !path.match(".*\\.tar");
},
},
["."],
);
// 使用統一的封裝管理器建立 .tar不壓縮
const outputTar = await createJobArchive(outputPath, jobId);
return Bun.file(outputTar);
},
{

178
src/pages/downloadChunk.tsx Normal file
View file

@ -0,0 +1,178 @@
/**
* Contents.CN Chunk API
*
*
*/
import { Elysia, t } from "elysia";
import { join } from "node:path";
import { outputDir } from "..";
import db from "../db/db";
import { WEBROOT } from "../helpers/env";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
createChunkDownloadHeaders,
getFileForDirectDownload,
createDirectDownloadHeaders,
CHUNK_SIZE_BYTES,
} from "../transfer";
import { statSync, existsSync } from "node:fs";
export const downloadChunk = new Elysia()
.use(userService)
/**
*
*/
.get(
"/download/:userId/:jobId/:fileName/info",
async ({ params, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const fileName = sanitize(decodeURIComponent(params.fileName));
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
const info = getChunkDownloadInfo(filePath);
if (!info) {
return { error: "File not found" };
}
return {
...info,
use_chunked: shouldUseChunkedDownload(filePath),
};
},
{ auth: true }
)
/**
* chunk
*/
.get(
"/download/:userId/:jobId/:fileName/chunk/:chunkIndex",
async ({ params, set, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
set.status = 404;
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const fileName = sanitize(decodeURIComponent(params.fileName));
const chunkIndex = parseInt(params.chunkIndex, 10);
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
const info = getChunkDownloadInfo(filePath);
if (!info) {
set.status = 404;
return { error: "File not found" };
}
const chunkData = await getChunk(filePath, chunkIndex);
if (!chunkData) {
set.status = 404;
return { error: "Chunk not found" };
}
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
for (const [key, value] of Object.entries(headers)) {
set.headers[key] = value;
}
return new Response(chunkData);
},
{ auth: true }
)
/**
* Archive chunk
*/
.get(
"/archive/:jobId/info",
async ({ params, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
if (!existsSync(archivePath)) {
return { error: "Archive not found" };
}
const info = getChunkDownloadInfo(archivePath);
if (!info) {
return { error: "Archive not found" };
}
return {
...info,
use_chunked: shouldUseChunkedDownload(archivePath),
};
},
{ auth: true }
)
/**
* Archive chunk
*/
.get(
"/archive/:jobId/chunk/:chunkIndex",
async ({ params, set, user }) => {
const userId = user.id;
const job = await db
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
.get(user.id, params.jobId);
if (!job) {
set.status = 404;
return { error: "Job not found" };
}
const jobId = decodeURIComponent(params.jobId);
const chunkIndex = parseInt(params.chunkIndex, 10);
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
const info = getChunkDownloadInfo(archivePath);
if (!info) {
set.status = 404;
return { error: "Archive not found" };
}
const chunkData = await getChunk(archivePath, chunkIndex);
if (!chunkData) {
set.status = 404;
return { error: "Chunk not found" };
}
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
for (const [key, value] of Object.entries(headers)) {
set.headers[key] = value;
}
return new Response(chunkData);
},
{ auth: true }
);

88
src/pages/uploadChunk.tsx Normal file
View file

@ -0,0 +1,88 @@
/**
* Contents.CN Chunk API
*
*
*/
import { Elysia, t } from "elysia";
import { WEBROOT } from "../helpers/env";
import { uploadsDir } from "../index";
import { userService } from "./user";
import sanitize from "sanitize-filename";
import {
handleChunkUpload,
shouldUseChunkedUpload,
CHUNK_SIZE_BYTES,
calculateChunkCount,
} from "../transfer";
export const uploadChunk = new Elysia().use(userService).post(
"/upload-chunk",
async ({ body, user, cookie: { jobId } }) => {
if (!jobId?.value) {
return {
success: false,
message: "No active job session",
};
}
const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body;
const sanitizedFileName = sanitize(file_name);
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
// 取得 chunk 資料
const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk;
const result = await handleChunkUpload(
upload_id,
parseInt(chunk_index, 10),
parseInt(total_chunks, 10),
chunkData,
sanitizedFileName,
parseInt(total_size, 10),
user.id,
jobId.value,
`${uploadsDir}${user.id}/`,
userUploadsDir
);
return result;
},
{
body: t.Object({
upload_id: t.String(),
chunk_index: t.String(),
total_chunks: t.String(),
file_name: t.String(),
total_size: t.String(),
chunk: t.File(),
}),
auth: true,
}
);
/**
* chunk
*/
export const uploadInfo = new Elysia().use(userService).post(
"/upload-info",
async ({ body }) => {
const { file_size } = body;
const size = parseInt(file_size, 10);
const useChunked = shouldUseChunkedUpload(size);
return {
use_chunked: useChunked,
chunk_size: CHUNK_SIZE_BYTES,
total_chunks: useChunked ? calculateChunkCount(size) : 1,
};
},
{
body: t.Object({
file_size: t.String(),
}),
auth: true,
}
);

View file

@ -0,0 +1,154 @@
/**
* Contents.CN
*
*
* - .tar
* - .tar.gz, .tgz, .zip
*/
import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
import { join, basename } from "node:path";
import * as tar from "tar";
import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants";
/**
*
*/
export function validateArchiveFormat(fileName: string): boolean {
const lowerName = fileName.toLowerCase();
// 檢查是否使用禁止的格式
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
if (lowerName.endsWith(forbidden)) {
return false;
}
}
// 檢查是否是允許的格式
return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT);
}
/**
* 使 .tar
*/
export function getArchiveFileName(baseName: string): string {
const lowerName = baseName.toLowerCase();
// 移除任何禁止的副檔名
let cleanName = baseName;
for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
if (lowerName.endsWith(forbidden)) {
cleanName = baseName.slice(0, -forbidden.length);
break;
}
}
// 如果已經是 .tar 結尾,直接返回
if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) {
return cleanName;
}
// 否則加上 .tar
return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`;
}
/**
* .tar
*
* @param sourceDir -
* @param outputPath - .tar
* @param options -
*/
export async function createTarArchive(
sourceDir: string,
outputPath: string,
options: {
filter?: (path: string) => boolean;
prefix?: string;
} = {}
): Promise<string> {
// 強制確保輸出是 .tar 格式
const finalOutputPath = getArchiveFileName(outputPath);
// 確保輸出目錄存在
const outputDir = join(finalOutputPath, "..");
if (!existsSync(outputDir)) {
mkdirSync(outputDir, { recursive: true });
}
// 取得要封裝的檔案列表
const files = readdirSync(sourceDir);
// 預設過濾器:排除其他 tar 檔案
const defaultFilter = (path: string) => !path.match(/\.tar$/i);
const filter = options.filter || defaultFilter;
await tar.create(
{
file: finalOutputPath,
cwd: sourceDir,
filter: filter,
// 不使用 gzip 壓縮
gzip: false,
},
files.filter(f => filter(f))
);
console.log(`Created tar archive: ${finalOutputPath}`);
return finalOutputPath;
}
/**
* .tar
*
* @param outputDir -
* @param jobId - ID
*/
export async function createJobArchive(outputDir: string, jobId: string): Promise<string> {
const archiveName = `converted_files_${jobId}.tar`;
const archivePath = join(outputDir, archiveName);
await createTarArchive(outputDir, archivePath, {
filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案
});
return archivePath;
}
/**
* MinerU
*
* @param mineruOutputDir - MinerU
* @param targetPath - .tar
*/
export async function createConverterArchive(
converterOutputDir: string,
targetPath: string
): Promise<string> {
// 確保輸出使用 .tar 格式
const tarPath = getArchiveFileName(targetPath);
await createTarArchive(converterOutputDir, tarPath);
return tarPath;
}
/**
*
*/
export function getArchiveInfo(archivePath: string): {
exists: boolean;
size: number;
fileName: string;
} | null {
if (!existsSync(archivePath)) {
return null;
}
const stats = statSync(archivePath);
return {
exists: true,
size: stats.size,
fileName: basename(archivePath),
};
}

38
src/transfer/constants.ts Normal file
View file

@ -0,0 +1,38 @@
/**
* Contents.CN
*
*
*/
/**
* 10MB
* -
* - 使 chunk
*/
export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
/**
* chunk 5MB
*/
export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
/**
* session
* 1
*/
export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
/**
*
*/
export const CHUNK_TEMP_DIR = "chunks_temp";
/**
* .tar
*/
export const ALLOWED_ARCHIVE_FORMAT = ".tar";
/**
*
*/
export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"];

View file

@ -0,0 +1,123 @@
/**
* Contents.CN
*
*
* - 10MB
* - >10MBchunk
*/
import { existsSync, statSync, createReadStream } from "node:fs";
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants";
import type { ChunkDownloadInfo } from "./types";
import { getTransferMode } from "./types";
import { basename } from "node:path";
/**
* 使 chunk
*/
export function shouldUseChunkedDownload(filePath: string): boolean {
if (!existsSync(filePath)) {
return false;
}
const stats = statSync(filePath);
return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked";
}
/**
* chunk
*/
export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null {
if (!existsSync(filePath)) {
return null;
}
const stats = statSync(filePath);
const fileName = basename(filePath);
return {
total_size: stats.size,
total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES),
chunk_size: CHUNK_SIZE_BYTES,
file_name: fileName,
};
}
/**
* chunk
*/
export async function getChunk(
filePath: string,
chunkIndex: number,
chunkSize: number = CHUNK_SIZE_BYTES
): Promise<Buffer | null> {
if (!existsSync(filePath)) {
return null;
}
const stats = statSync(filePath);
const start = chunkIndex * chunkSize;
const end = Math.min(start + chunkSize, stats.size);
if (start >= stats.size) {
return null;
}
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
const stream = createReadStream(filePath, { start, end: end - 1 });
stream.on("data", (chunk) => {
chunks.push(chunk as Buffer);
});
stream.on("end", () => {
resolve(Buffer.concat(chunks));
});
stream.on("error", reject);
});
}
/**
*
*/
export function getFileForDirectDownload(filePath: string): ReturnType<typeof Bun.file> | null {
if (!existsSync(filePath)) {
return null;
}
return Bun.file(filePath);
}
/**
* chunk headers
*/
export function createChunkDownloadHeaders(
info: ChunkDownloadInfo,
chunkIndex: number,
chunkData: Buffer
): Record<string, string> {
const start = chunkIndex * info.chunk_size;
const end = start + chunkData.length - 1;
return {
"Content-Type": "application/octet-stream",
"Content-Length": chunkData.length.toString(),
"Content-Range": `bytes ${start}-${end}/${info.total_size}`,
"Accept-Ranges": "bytes",
"X-Chunk-Index": chunkIndex.toString(),
"X-Total-Chunks": info.total_chunks.toString(),
"X-File-Name": encodeURIComponent(info.file_name),
"X-Total-Size": info.total_size.toString(),
};
}
/**
* headers
*/
export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record<string, string> {
return {
"Content-Type": "application/octet-stream",
"Content-Length": fileSize.toString(),
"Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`,
};
}

56
src/transfer/index.ts Normal file
View file

@ -0,0 +1,56 @@
/**
* Contents.CN
*
*
*/
// 常數
export {
CHUNK_THRESHOLD_BYTES,
CHUNK_SIZE_BYTES,
UPLOAD_SESSION_TIMEOUT_MS,
CHUNK_TEMP_DIR,
ALLOWED_ARCHIVE_FORMAT,
FORBIDDEN_ARCHIVE_FORMATS,
} from "./constants";
// 類型
export type {
ChunkUploadRequest,
ChunkUploadResponse,
DirectUploadResponse,
ChunkDownloadInfo,
UploadSession,
TransferMode,
} from "./types";
export { getTransferMode } from "./types";
// 上傳管理
export {
uploadSessionManager,
shouldUseChunkedUpload,
handleDirectUpload,
handleChunkUpload,
calculateChunkCount,
} from "./uploadManager";
// 下載管理
export {
shouldUseChunkedDownload,
getChunkDownloadInfo,
getChunk,
getFileForDirectDownload,
createChunkDownloadHeaders,
createDirectDownloadHeaders,
} from "./downloadManager";
// 封裝管理
export {
validateArchiveFormat,
getArchiveFileName,
createTarArchive,
createJobArchive,
createConverterArchive,
getArchiveInfo,
} from "./archiveManager";

94
src/transfer/types.ts Normal file
View file

@ -0,0 +1,94 @@
/**
* Contents.CN
*/
/**
* Chunk
*/
export interface ChunkUploadRequest {
/** 上傳會話 IDUUID */
upload_id: string;
/** 當前 chunk 索引(從 0 開始) */
chunk_index: number;
/** 總 chunk 數量 */
total_chunks: number;
/** chunk 數據 */
data: Blob | ArrayBuffer;
/** 原始檔案名稱 */
file_name: string;
/** 檔案總大小 */
total_size: number;
}
/**
* Chunk
*/
export interface ChunkUploadResponse {
success: boolean;
message: string;
/** 已接收的 chunks 索引列表 */
received_chunks?: number[];
/** 是否所有 chunks 都已接收完成 */
completed?: boolean;
/** 合併後的檔案路徑(僅完成時返回) */
file_path?: string;
}
/**
*
*/
export interface DirectUploadResponse {
success: boolean;
message: string;
file_path?: string;
}
/**
* Chunk
*/
export interface ChunkDownloadInfo {
/** 檔案總大小 */
total_size: number;
/** chunk 數量 */
total_chunks: number;
/** 每個 chunk 大小 */
chunk_size: number;
/** 檔案名稱 */
file_name: string;
}
/**
*
*/
export interface UploadSession {
/** 上傳 ID */
upload_id: string;
/** 使用者 ID */
user_id: string;
/** Job ID */
job_id: string;
/** 檔案名稱 */
file_name: string;
/** 檔案總大小 */
total_size: number;
/** 總 chunk 數 */
total_chunks: number;
/** 已接收的 chunks */
received_chunks: Set<number>;
/** 建立時間 */
created_at: Date;
/** 暫存目錄路徑 */
temp_dir: string;
}
/**
*
*/
export type TransferMode = "direct" | "chunked";
/**
*
*/
export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode {
return fileSize <= thresholdBytes ? "direct" : "chunked";
}

View file

@ -0,0 +1,303 @@
/**
* Contents.CN
*
*
* - 10MB
* - >10MBchunk
*/
import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs";
import { join } from "node:path";
import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants";
import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types";
import { getTransferMode } from "./types";
/**
*
* chunk
*/
class UploadSessionManager {
private sessions: Map<string, UploadSession> = new Map();
private cleanupInterval: ReturnType<typeof setInterval> | null = null;
constructor() {
// 定期清理過期的 sessions
this.cleanupInterval = setInterval(() => {
this.cleanupExpiredSessions();
}, 5 * 60 * 1000); // 每 5 分鐘清理一次
}
/**
*
*/
createSession(
uploadId: string,
userId: string,
jobId: string,
fileName: string,
totalSize: number,
totalChunks: number,
baseTempDir: string
): UploadSession {
const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId);
if (!existsSync(tempDir)) {
mkdirSync(tempDir, { recursive: true });
}
const session: UploadSession = {
upload_id: uploadId,
user_id: userId,
job_id: jobId,
file_name: fileName,
total_size: totalSize,
total_chunks: totalChunks,
received_chunks: new Set(),
created_at: new Date(),
temp_dir: tempDir,
};
this.sessions.set(uploadId, session);
return session;
}
/**
*
*/
getSession(uploadId: string): UploadSession | undefined {
return this.sessions.get(uploadId);
}
/**
* chunk
*/
markChunkReceived(uploadId: string, chunkIndex: number): boolean {
const session = this.sessions.get(uploadId);
if (!session) return false;
session.received_chunks.add(chunkIndex);
return true;
}
/**
* chunks
*/
isComplete(uploadId: string): boolean {
const session = this.sessions.get(uploadId);
if (!session) return false;
return session.received_chunks.size === session.total_chunks;
}
/**
*
*/
removeSession(uploadId: string): void {
const session = this.sessions.get(uploadId);
if (session && existsSync(session.temp_dir)) {
rmSync(session.temp_dir, { recursive: true, force: true });
}
this.sessions.delete(uploadId);
}
/**
*
*/
private cleanupExpiredSessions(): void {
const now = Date.now();
for (const [uploadId, session] of this.sessions) {
if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) {
console.log(`Cleaning up expired upload session: ${uploadId}`);
this.removeSession(uploadId);
}
}
}
/**
*
*/
destroy(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = null;
}
}
}
// 單例實例
export const uploadSessionManager = new UploadSessionManager();
/**
* 使 chunk
*/
export function shouldUseChunkedUpload(fileSize: number): boolean {
return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked";
}
/**
*
*/
export async function handleDirectUpload(
file: File,
targetDir: string,
fileName: string
): Promise<DirectUploadResponse> {
try {
const targetPath = join(targetDir, fileName);
if (!existsSync(targetDir)) {
mkdirSync(targetDir, { recursive: true });
}
await Bun.write(targetPath, file);
return {
success: true,
message: "File uploaded successfully.",
file_path: targetPath,
};
} catch (error) {
console.error("Direct upload failed:", error);
return {
success: false,
message: `Upload failed: ${error}`,
};
}
}
/**
* chunk
*/
export async function handleChunkUpload(
uploadId: string,
chunkIndex: number,
totalChunks: number,
chunkData: ArrayBuffer | Blob | Buffer,
fileName: string,
totalSize: number,
userId: string,
jobId: string,
baseTempDir: string,
targetDir: string
): Promise<ChunkUploadResponse> {
try {
// 取得或建立會話
let session = uploadSessionManager.getSession(uploadId);
if (!session) {
session = uploadSessionManager.createSession(
uploadId,
userId,
jobId,
fileName,
totalSize,
totalChunks,
baseTempDir
);
}
// 驗證會話資訊一致性
if (session.total_chunks !== totalChunks || session.file_name !== fileName) {
return {
success: false,
message: "Session mismatch: inconsistent upload parameters",
};
}
// 儲存 chunk
const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`);
// 處理不同類型的 chunk 資料
let data: ArrayBuffer | Buffer;
if (chunkData instanceof Blob) {
data = await chunkData.arrayBuffer();
} else if (Buffer.isBuffer(chunkData)) {
data = chunkData;
} else {
data = chunkData;
}
await Bun.write(chunkPath, data);
// 標記已接收
uploadSessionManager.markChunkReceived(uploadId, chunkIndex);
// 檢查是否完成
if (uploadSessionManager.isComplete(uploadId)) {
// 合併所有 chunks
const finalPath = await mergeChunks(session, targetDir);
// 清理會話
uploadSessionManager.removeSession(uploadId);
return {
success: true,
message: "Upload completed and merged successfully.",
received_chunks: Array.from({ length: totalChunks }, (_, i) => i),
completed: true,
file_path: finalPath,
};
}
return {
success: true,
message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`,
received_chunks: Array.from(session.received_chunks),
completed: false,
};
} catch (error) {
console.error("Chunk upload failed:", error);
return {
success: false,
message: `Chunk upload failed: ${error}`,
};
}
}
/**
* chunks
*/
async function mergeChunks(session: UploadSession, targetDir: string): Promise<string> {
const targetPath = join(targetDir, session.file_name);
if (!existsSync(targetDir)) {
mkdirSync(targetDir, { recursive: true });
}
// 讀取並排序所有 chunks
const chunkFiles = readdirSync(session.temp_dir)
.filter(f => f.startsWith("chunk_"))
.sort();
// 建立輸出串流
const writeStream = createWriteStream(targetPath);
// 依序寫入每個 chunk
for (const chunkFile of chunkFiles) {
const chunkPath = join(session.temp_dir, chunkFile);
const chunkData = await Bun.file(chunkPath).arrayBuffer();
writeStream.write(Buffer.from(chunkData));
}
// 結束寫入
await new Promise<void>((resolve, reject) => {
writeStream.end((err: Error | null) => {
if (err) reject(err);
else resolve();
});
});
// 驗證檔案大小
const finalStats = statSync(targetPath);
if (finalStats.size !== session.total_size) {
console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`);
}
console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`);
return targetPath;
}
/**
* chunk
*/
export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
return Math.ceil(fileSize / chunkSize);
}