feat: 實作全域檔案傳輸機制與 .tar 封裝規範

## 主要變更

### 新增全域檔案傳輸模組 (src/transfer/)
- constants.ts: 定義傳輸常數（10MB 門檻、5MB chunk 大小）
- types.ts: 傳輸類型定義
- uploadManager.ts: 後端上傳管理器（支援直傳與 chunk）
- downloadManager.ts: 後端下載管理器（支援直傳與 chunk）
- archiveManager.ts: 封裝管理器（僅允許 .tar）
- index.ts: 統一匯出

### 新增 API 端點
- uploadChunk.tsx: Chunk 上傳 API
- downloadChunk.tsx: Chunk 下載 API

### 前端更新
- public/script.js: 整合智慧傳輸策略（≤10MB 直傳，>10MB chunk）
- public/transfer.js: 前端傳輸管理模組

### 轉換器更新
- mineru.ts: 改用 .tar 格式（不壓縮），禁止 .tar.gz
- download.tsx: 使用統一的封裝管理器

### 測試
- tests/transfer/: 完整傳輸機制測試套件
- tests/converters/mineru.test.ts: 更新以符合 .tar 規範

### 文件
- README.md: 新增檔案傳輸機制說明

## 設計原則
- 檔案 ≤10MB：直接傳輸
- 檔案 >10MB：使用 5MB chunks 分段傳輸
- 多檔輸出：僅允許 .tar 封裝（禁止 .tar.gz/.zip）
- 引擎層不感知 chunk（僅傳輸層處理）

src/transfer/archiveManager.ts

@@ -0,0 +1,154 @@
+/**
+ * Contents.CN 封裝管理器
+ * 
+ * 統一處理多檔輸出的封裝：
+ * - 唯一允許格式：.tar
+ * - 禁止：.tar.gz, .tgz, .zip 等
+ */
+
+import { existsSync, mkdirSync, readdirSync, statSync } from "node:fs";
+import { join, basename } from "node:path";
+import * as tar from "tar";
+import { ALLOWED_ARCHIVE_FORMAT, FORBIDDEN_ARCHIVE_FORMATS } from "./constants";
+
+/**
+ * 驗證封裝格式是否合法
+ */
+export function validateArchiveFormat(fileName: string): boolean {
+  const lowerName = fileName.toLowerCase();
+  
+  // 檢查是否使用禁止的格式
+  for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
+    if (lowerName.endsWith(forbidden)) {
+      return false;
+    }
+  }
+  
+  // 檢查是否是允許的格式
+  return lowerName.endsWith(ALLOWED_ARCHIVE_FORMAT);
+}
+
+/**
+ * 取得正確的封裝檔名（強制使用 .tar）
+ */
+export function getArchiveFileName(baseName: string): string {
+  const lowerName = baseName.toLowerCase();
+  
+  // 移除任何禁止的副檔名
+  let cleanName = baseName;
+  for (const forbidden of FORBIDDEN_ARCHIVE_FORMATS) {
+    if (lowerName.endsWith(forbidden)) {
+      cleanName = baseName.slice(0, -forbidden.length);
+      break;
+    }
+  }
+  
+  // 如果已經是 .tar 結尾，直接返回
+  if (cleanName.toLowerCase().endsWith(ALLOWED_ARCHIVE_FORMAT)) {
+    return cleanName;
+  }
+  
+  // 否則加上 .tar
+  return `${cleanName}${ALLOWED_ARCHIVE_FORMAT}`;
+}
+
+/**
+ * 建立 .tar 封裝（不壓縮）
+ * 
+ * @param sourceDir - 來源目錄
+ * @param outputPath - 輸出的 .tar 檔案路徑
+ * @param options - 額外選項
+ */
+export async function createTarArchive(
+  sourceDir: string,
+  outputPath: string,
+  options: {
+    filter?: (path: string) => boolean;
+    prefix?: string;
+  } = {}
+): Promise<string> {
+  // 強制確保輸出是 .tar 格式
+  const finalOutputPath = getArchiveFileName(outputPath);
+  
+  // 確保輸出目錄存在
+  const outputDir = join(finalOutputPath, "..");
+  if (!existsSync(outputDir)) {
+    mkdirSync(outputDir, { recursive: true });
+  }
+
+  // 取得要封裝的檔案列表
+  const files = readdirSync(sourceDir);
+  
+  // 預設過濾器：排除其他 tar 檔案
+  const defaultFilter = (path: string) => !path.match(/\.tar$/i);
+  const filter = options.filter || defaultFilter;
+
+  await tar.create(
+    {
+      file: finalOutputPath,
+      cwd: sourceDir,
+      filter: filter,
+      // 不使用 gzip 壓縮
+      gzip: false,
+    },
+    files.filter(f => filter(f))
+  );
+
+  console.log(`Created tar archive: ${finalOutputPath}`);
+  return finalOutputPath;
+}
+
+/**
+ * 建立用於多檔輸出的 .tar 封裝
+ * 
+ * @param outputDir - 輸出目錄（包含多個轉換結果）
+ * @param jobId - 任務 ID
+ */
+export async function createJobArchive(outputDir: string, jobId: string): Promise<string> {
+  const archiveName = `converted_files_${jobId}.tar`;
+  const archivePath = join(outputDir, archiveName);
+
+  await createTarArchive(outputDir, archivePath, {
+    filter: (path) => !path.endsWith(".tar"), // 排除其他 tar 檔案
+  });
+
+  return archivePath;
+}
+
+/**
+ * 從目錄封裝 MinerU 等多檔輸出引擎的結果
+ * 
+ * @param mineruOutputDir - MinerU 輸出目錄
+ * @param targetPath - 目標 .tar 路徑
+ */
+export async function createConverterArchive(
+  converterOutputDir: string,
+  targetPath: string
+): Promise<string> {
+  // 確保輸出使用 .tar 格式
+  const tarPath = getArchiveFileName(targetPath);
+  
+  await createTarArchive(converterOutputDir, tarPath);
+  
+  return tarPath;
+}
+
+/**
+ * 取得封裝檔案資訊
+ */
+export function getArchiveInfo(archivePath: string): {
+  exists: boolean;
+  size: number;
+  fileName: string;
+} | null {
+  if (!existsSync(archivePath)) {
+    return null;
+  }
+
+  const stats = statSync(archivePath);
+  return {
+    exists: true,
+    size: stats.size,
+    fileName: basename(archivePath),
+  };
+}

src/transfer/constants.ts

@@ -0,0 +1,38 @@
+/**
+ * Contents.CN 全域檔案傳輸常數
+ * 
+ * 這些常數定義了整個系統的檔案傳輸規則
+ */
+
+/**
+ * 檔案大小門檻（10MB）
+ * - 小於等於此值：直接傳輸
+ * - 大於此值：使用 chunk 分段傳輸
+ */
+export const CHUNK_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB
+
+/**
+ * 每個 chunk 的大小（5MB）
+ */
+export const CHUNK_SIZE_BYTES = 5 * 1024 * 1024; // 5MB
+
+/**
+ * 上傳 session 過期時間（毫秒）
+ * 預設 1 小時
+ */
+export const UPLOAD_SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
+
+/**
+ * 暫存目錄名稱
+ */
+export const CHUNK_TEMP_DIR = "chunks_temp";
+
+/**
+ * 允許的封裝格式（僅 .tar）
+ */
+export const ALLOWED_ARCHIVE_FORMAT = ".tar";
+
+/**
+ * 禁止的封裝格式
+ */
+export const FORBIDDEN_ARCHIVE_FORMATS = [".tar.gz", ".tgz", ".zip", ".gz"];

src/transfer/downloadManager.ts

@@ -0,0 +1,123 @@
+/**
+ * Contents.CN 後端下載管理器
+ * 
+ * 統一處理所有檔案下載，包含：
+ * - 小檔（≤10MB）：直接回傳
+ * - 大檔（>10MB）：chunk 分段下載
+ */
+
+import { existsSync, statSync, createReadStream } from "node:fs";
+import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES } from "./constants";
+import type { ChunkDownloadInfo } from "./types";
+import { getTransferMode } from "./types";
+import { basename } from "node:path";
+
+/**
+ * 判斷檔案是否需要使用 chunk 下載
+ */
+export function shouldUseChunkedDownload(filePath: string): boolean {
+  if (!existsSync(filePath)) {
+    return false;
+  }
+  const stats = statSync(filePath);
+  return getTransferMode(stats.size, CHUNK_THRESHOLD_BYTES) === "chunked";
+}
+
+/**
+ * 取得檔案的 chunk 下載資訊
+ */
+export function getChunkDownloadInfo(filePath: string): ChunkDownloadInfo | null {
+  if (!existsSync(filePath)) {
+    return null;
+  }
+
+  const stats = statSync(filePath);
+  const fileName = basename(filePath);
+
+  return {
+    total_size: stats.size,
+    total_chunks: Math.ceil(stats.size / CHUNK_SIZE_BYTES),
+    chunk_size: CHUNK_SIZE_BYTES,
+    file_name: fileName,
+  };
+}
+
+/**
+ * 取得特定 chunk 的資料
+ */
+export async function getChunk(
+  filePath: string, 
+  chunkIndex: number, 
+  chunkSize: number = CHUNK_SIZE_BYTES
+): Promise<Buffer | null> {
+  if (!existsSync(filePath)) {
+    return null;
+  }
+
+  const stats = statSync(filePath);
+  const start = chunkIndex * chunkSize;
+  const end = Math.min(start + chunkSize, stats.size);
+
+  if (start >= stats.size) {
+    return null;
+  }
+
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    const stream = createReadStream(filePath, { start, end: end - 1 });
+
+    stream.on("data", (chunk) => {
+      chunks.push(chunk as Buffer);
+    });
+
+    stream.on("end", () => {
+      resolve(Buffer.concat(chunks));
+    });
+
+    stream.on("error", reject);
+  });
+}
+
+/**
+ * 直接取得完整檔案（小檔用）
+ */
+export function getFileForDirectDownload(filePath: string): ReturnType<typeof Bun.file> | null {
+  if (!existsSync(filePath)) {
+    return null;
+  }
+  return Bun.file(filePath);
+}
+
+/**
+ * 建立 chunk 下載回應的 headers
+ */
+export function createChunkDownloadHeaders(
+  info: ChunkDownloadInfo, 
+  chunkIndex: number, 
+  chunkData: Buffer
+): Record<string, string> {
+  const start = chunkIndex * info.chunk_size;
+  const end = start + chunkData.length - 1;
+
+  return {
+    "Content-Type": "application/octet-stream",
+    "Content-Length": chunkData.length.toString(),
+    "Content-Range": `bytes ${start}-${end}/${info.total_size}`,
+    "Accept-Ranges": "bytes",
+    "X-Chunk-Index": chunkIndex.toString(),
+    "X-Total-Chunks": info.total_chunks.toString(),
+    "X-File-Name": encodeURIComponent(info.file_name),
+    "X-Total-Size": info.total_size.toString(),
+  };
+}
+
+/**
+ * 建立直接下載回應的 headers
+ */
+export function createDirectDownloadHeaders(fileName: string, fileSize: number): Record<string, string> {
+  return {
+    "Content-Type": "application/octet-stream",
+    "Content-Length": fileSize.toString(),
+    "Content-Disposition": `attachment; filename="${encodeURIComponent(fileName)}"`,
+  };
+}

src/transfer/index.ts

@@ -0,0 +1,56 @@
+/**
+ * Contents.CN 全域檔案傳輸模組
+ * 
+ * 統一匯出所有傳輸相關功能
+ */
+
+// 常數
+export {
+  CHUNK_THRESHOLD_BYTES,
+  CHUNK_SIZE_BYTES,
+  UPLOAD_SESSION_TIMEOUT_MS,
+  CHUNK_TEMP_DIR,
+  ALLOWED_ARCHIVE_FORMAT,
+  FORBIDDEN_ARCHIVE_FORMATS,
+} from "./constants";
+
+// 類型
+export type {
+  ChunkUploadRequest,
+  ChunkUploadResponse,
+  DirectUploadResponse,
+  ChunkDownloadInfo,
+  UploadSession,
+  TransferMode,
+} from "./types";
+
+export { getTransferMode } from "./types";
+
+// 上傳管理
+export {
+  uploadSessionManager,
+  shouldUseChunkedUpload,
+  handleDirectUpload,
+  handleChunkUpload,
+  calculateChunkCount,
+} from "./uploadManager";
+
+// 下載管理
+export {
+  shouldUseChunkedDownload,
+  getChunkDownloadInfo,
+  getChunk,
+  getFileForDirectDownload,
+  createChunkDownloadHeaders,
+  createDirectDownloadHeaders,
+} from "./downloadManager";
+
+// 封裝管理
+export {
+  validateArchiveFormat,
+  getArchiveFileName,
+  createTarArchive,
+  createJobArchive,
+  createConverterArchive,
+  getArchiveInfo,
+} from "./archiveManager";

src/transfer/types.ts

@@ -0,0 +1,94 @@
+/**
+ * Contents.CN 檔案傳輸類型定義
+ */
+
+/**
+ * Chunk 上傳請求結構
+ */
+export interface ChunkUploadRequest {
+  /** 上傳會話 ID（UUID） */
+  upload_id: string;
+  /** 當前 chunk 索引（從 0 開始） */
+  chunk_index: number;
+  /** 總 chunk 數量 */
+  total_chunks: number;
+  /** chunk 數據 */
+  data: Blob | ArrayBuffer;
+  /** 原始檔案名稱 */
+  file_name: string;
+  /** 檔案總大小 */
+  total_size: number;
+}
+
+/**
+ * Chunk 上傳回應結構
+ */
+export interface ChunkUploadResponse {
+  success: boolean;
+  message: string;
+  /** 已接收的 chunks 索引列表 */
+  received_chunks?: number[];
+  /** 是否所有 chunks 都已接收完成 */
+  completed?: boolean;
+  /** 合併後的檔案路徑（僅完成時返回） */
+  file_path?: string;
+}
+
+/**
+ * 直接上傳回應結構
+ */
+export interface DirectUploadResponse {
+  success: boolean;
+  message: string;
+  file_path?: string;
+}
+
+/**
+ * Chunk 下載資訊
+ */
+export interface ChunkDownloadInfo {
+  /** 檔案總大小 */
+  total_size: number;
+  /** chunk 數量 */
+  total_chunks: number;
+  /** 每個 chunk 大小 */
+  chunk_size: number;
+  /** 檔案名稱 */
+  file_name: string;
+}
+
+/**
+ * 上傳會話狀態
+ */
+export interface UploadSession {
+  /** 上傳 ID */
+  upload_id: string;
+  /** 使用者 ID */
+  user_id: string;
+  /** Job ID */
+  job_id: string;
+  /** 檔案名稱 */
+  file_name: string;
+  /** 檔案總大小 */
+  total_size: number;
+  /** 總 chunk 數 */
+  total_chunks: number;
+  /** 已接收的 chunks */
+  received_chunks: Set<number>;
+  /** 建立時間 */
+  created_at: Date;
+  /** 暫存目錄路徑 */
+  temp_dir: string;
+}
+
+/**
+ * 傳輸模式
+ */
+export type TransferMode = "direct" | "chunked";
+
+/**
+ * 根據檔案大小決定傳輸模式
+ */
+export function getTransferMode(fileSize: number, thresholdBytes: number): TransferMode {
+  return fileSize <= thresholdBytes ? "direct" : "chunked";
+}

src/transfer/uploadManager.ts

@@ -0,0 +1,303 @@
+/**
+ * Contents.CN 後端上傳管理器
+ * 
+ * 統一處理所有檔案上傳，包含：
+ * - 小檔（≤10MB）：直接接收
+ * - 大檔（>10MB）：chunk 接收與合併
+ */
+
+import { existsSync, mkdirSync, rmSync, readdirSync, createWriteStream, statSync } from "node:fs";
+import { join } from "node:path";
+import { CHUNK_THRESHOLD_BYTES, CHUNK_SIZE_BYTES, UPLOAD_SESSION_TIMEOUT_MS, CHUNK_TEMP_DIR } from "./constants";
+import type { UploadSession, ChunkUploadResponse, DirectUploadResponse } from "./types";
+import { getTransferMode } from "./types";
+
+/**
+ * 上傳會話管理器
+ * 用於追蹤進行中的 chunk 上傳
+ */
+class UploadSessionManager {
+  private sessions: Map<string, UploadSession> = new Map();
+  private cleanupInterval: ReturnType<typeof setInterval> | null = null;
+
+  constructor() {
+    // 定期清理過期的 sessions
+    this.cleanupInterval = setInterval(() => {
+      this.cleanupExpiredSessions();
+    }, 5 * 60 * 1000); // 每 5 分鐘清理一次
+  }
+
+  /**
+   * 建立新的上傳會話
+   */
+  createSession(
+    uploadId: string,
+    userId: string,
+    jobId: string,
+    fileName: string,
+    totalSize: number,
+    totalChunks: number,
+    baseTempDir: string
+  ): UploadSession {
+    const tempDir = join(baseTempDir, CHUNK_TEMP_DIR, uploadId);
+    
+    if (!existsSync(tempDir)) {
+      mkdirSync(tempDir, { recursive: true });
+    }
+
+    const session: UploadSession = {
+      upload_id: uploadId,
+      user_id: userId,
+      job_id: jobId,
+      file_name: fileName,
+      total_size: totalSize,
+      total_chunks: totalChunks,
+      received_chunks: new Set(),
+      created_at: new Date(),
+      temp_dir: tempDir,
+    };
+
+    this.sessions.set(uploadId, session);
+    return session;
+  }
+
+  /**
+   * 取得上傳會話
+   */
+  getSession(uploadId: string): UploadSession | undefined {
+    return this.sessions.get(uploadId);
+  }
+
+  /**
+   * 更新已接收的 chunk
+   */
+  markChunkReceived(uploadId: string, chunkIndex: number): boolean {
+    const session = this.sessions.get(uploadId);
+    if (!session) return false;
+    
+    session.received_chunks.add(chunkIndex);
+    return true;
+  }
+
+  /**
+   * 檢查是否所有 chunks 都已接收
+   */
+  isComplete(uploadId: string): boolean {
+    const session = this.sessions.get(uploadId);
+    if (!session) return false;
+    
+    return session.received_chunks.size === session.total_chunks;
+  }
+
+  /**
+   * 移除會話並清理暫存檔案
+   */
+  removeSession(uploadId: string): void {
+    const session = this.sessions.get(uploadId);
+    if (session && existsSync(session.temp_dir)) {
+      rmSync(session.temp_dir, { recursive: true, force: true });
+    }
+    this.sessions.delete(uploadId);
+  }
+
+  /**
+   * 清理過期的會話
+   */
+  private cleanupExpiredSessions(): void {
+    const now = Date.now();
+    for (const [uploadId, session] of this.sessions) {
+      if (now - session.created_at.getTime() > UPLOAD_SESSION_TIMEOUT_MS) {
+        console.log(`Cleaning up expired upload session: ${uploadId}`);
+        this.removeSession(uploadId);
+      }
+    }
+  }
+
+  /**
+   * 停止清理計時器
+   */
+  destroy(): void {
+    if (this.cleanupInterval) {
+      clearInterval(this.cleanupInterval);
+      this.cleanupInterval = null;
+    }
+  }
+}
+
+// 單例實例
+export const uploadSessionManager = new UploadSessionManager();
+
+/**
+ * 判斷檔案是否需要使用 chunk 上傳
+ */
+export function shouldUseChunkedUpload(fileSize: number): boolean {
+  return getTransferMode(fileSize, CHUNK_THRESHOLD_BYTES) === "chunked";
+}
+
+/**
+ * 處理直接上傳（小檔）
+ */
+export async function handleDirectUpload(
+  file: File,
+  targetDir: string,
+  fileName: string
+): Promise<DirectUploadResponse> {
+  try {
+    const targetPath = join(targetDir, fileName);
+    
+    if (!existsSync(targetDir)) {
+      mkdirSync(targetDir, { recursive: true });
+    }
+
+    await Bun.write(targetPath, file);
+
+    return {
+      success: true,
+      message: "File uploaded successfully.",
+      file_path: targetPath,
+    };
+  } catch (error) {
+    console.error("Direct upload failed:", error);
+    return {
+      success: false,
+      message: `Upload failed: ${error}`,
+    };
+  }
+}
+
+/**
+ * 處理 chunk 上傳
+ */
+export async function handleChunkUpload(
+  uploadId: string,
+  chunkIndex: number,
+  totalChunks: number,
+  chunkData: ArrayBuffer | Blob | Buffer,
+  fileName: string,
+  totalSize: number,
+  userId: string,
+  jobId: string,
+  baseTempDir: string,
+  targetDir: string
+): Promise<ChunkUploadResponse> {
+  try {
+    // 取得或建立會話
+    let session = uploadSessionManager.getSession(uploadId);
+    
+    if (!session) {
+      session = uploadSessionManager.createSession(
+        uploadId,
+        userId,
+        jobId,
+        fileName,
+        totalSize,
+        totalChunks,
+        baseTempDir
+      );
+    }
+
+    // 驗證會話資訊一致性
+    if (session.total_chunks !== totalChunks || session.file_name !== fileName) {
+      return {
+        success: false,
+        message: "Session mismatch: inconsistent upload parameters",
+      };
+    }
+
+    // 儲存 chunk
+    const chunkPath = join(session.temp_dir, `chunk_${chunkIndex.toString().padStart(6, "0")}`);
+    // 處理不同類型的 chunk 資料
+    let data: ArrayBuffer | Buffer;
+    if (chunkData instanceof Blob) {
+      data = await chunkData.arrayBuffer();
+    } else if (Buffer.isBuffer(chunkData)) {
+      data = chunkData;
+    } else {
+      data = chunkData;
+    }
+    await Bun.write(chunkPath, data);
+
+    // 標記已接收
+    uploadSessionManager.markChunkReceived(uploadId, chunkIndex);
+
+    // 檢查是否完成
+    if (uploadSessionManager.isComplete(uploadId)) {
+      // 合併所有 chunks
+      const finalPath = await mergeChunks(session, targetDir);
+      
+      // 清理會話
+      uploadSessionManager.removeSession(uploadId);
+
+      return {
+        success: true,
+        message: "Upload completed and merged successfully.",
+        received_chunks: Array.from({ length: totalChunks }, (_, i) => i),
+        completed: true,
+        file_path: finalPath,
+      };
+    }
+
+    return {
+      success: true,
+      message: `Chunk ${chunkIndex + 1}/${totalChunks} received.`,
+      received_chunks: Array.from(session.received_chunks),
+      completed: false,
+    };
+  } catch (error) {
+    console.error("Chunk upload failed:", error);
+    return {
+      success: false,
+      message: `Chunk upload failed: ${error}`,
+    };
+  }
+}
+
+/**
+ * 合併所有 chunks 為完整檔案
+ */
+async function mergeChunks(session: UploadSession, targetDir: string): Promise<string> {
+  const targetPath = join(targetDir, session.file_name);
+  
+  if (!existsSync(targetDir)) {
+    mkdirSync(targetDir, { recursive: true });
+  }
+
+  // 讀取並排序所有 chunks
+  const chunkFiles = readdirSync(session.temp_dir)
+    .filter(f => f.startsWith("chunk_"))
+    .sort();
+
+  // 建立輸出串流
+  const writeStream = createWriteStream(targetPath);
+
+  // 依序寫入每個 chunk
+  for (const chunkFile of chunkFiles) {
+    const chunkPath = join(session.temp_dir, chunkFile);
+    const chunkData = await Bun.file(chunkPath).arrayBuffer();
+    writeStream.write(Buffer.from(chunkData));
+  }
+
+  // 結束寫入
+  await new Promise<void>((resolve, reject) => {
+    writeStream.end((err: Error | null) => {
+      if (err) reject(err);
+      else resolve();
+    });
+  });
+
+  // 驗證檔案大小
+  const finalStats = statSync(targetPath);
+  if (finalStats.size !== session.total_size) {
+    console.warn(`File size mismatch: expected ${session.total_size}, got ${finalStats.size}`);
+  }
+
+  console.log(`Successfully merged ${chunkFiles.length} chunks into ${targetPath}`);
+  return targetPath;
+}
+
+/**
+ * 計算需要的 chunk 數量
+ */
+export function calculateChunkCount(fileSize: number, chunkSize: number = CHUNK_SIZE_BYTES): number {
+  return Math.ceil(fileSize / chunkSize);
+}