feat: 實作全域檔案傳輸機制與 .tar 封裝規範
## 主要變更 ### 新增全域檔案傳輸模組 (src/transfer/) - constants.ts: 定義傳輸常數（10MB 門檻、5MB chunk 大小） - types.ts: 傳輸類型定義 - uploadManager.ts: 後端上傳管理器（支援直傳與 chunk） - downloadManager.ts: 後端下載管理器（支援直傳與 chunk） - archiveManager.ts: 封裝管理器（僅允許 .tar） - index.ts: 統一匯出 ### 新增 API 端點 - uploadChunk.tsx: Chunk 上傳 API - downloadChunk.tsx: Chunk 下載 API ### 前端更新 - public/script.js: 整合智慧傳輸策略（≤10MB 直傳，>10MB chunk） - public/transfer.js: 前端傳輸管理模組 ### 轉換器更新 - mineru.ts: 改用 .tar 格式（不壓縮），禁止 .tar.gz - download.tsx: 使用統一的封裝管理器 ### 測試 - tests/transfer/: 完整傳輸機制測試套件 - tests/converters/mineru.test.ts: 更新以符合 .tar 規範 ### 文件 - README.md: 新增檔案傳輸機制說明 ## 設計原則 - 檔案 ≤10MB：直接傳輸 - 檔案 >10MB：使用 5MB chunks 分段傳輸 - 多檔輸出：僅允許 .tar 封裝（禁止 .tar.gz/.zip） - 引擎層不感知 chunk（僅傳輸層處理）
This commit is contained in:
Your Name
parent
f7ebc084ea
commit
5322f85721
18 changed files with 2387 additions and 61 deletions
|
|
@ -1,11 +1,11 @@
|
|||
import path from "node:path";
|
||||
import { Elysia } from "elysia";
|
||||
import sanitize from "sanitize-filename";
|
||||
import * as tar from "tar";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import { createJobArchive, shouldUseChunkedDownload } from "../transfer";
|
||||
|
||||
export const download = new Elysia()
|
||||
.use(userService)
|
||||
|
|
@ -45,18 +45,10 @@ export const download = new Elysia()
|
|||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const outputPath = `${outputDir}${userId}/${jobId}`;
|
||||
const outputTar = path.join(outputPath, `converted_files_${jobId}.tar`);
|
||||
|
||||
await tar.create(
|
||||
{
|
||||
file: outputTar,
|
||||
cwd: outputPath,
|
||||
filter: (path) => {
|
||||
return !path.match(".*\\.tar");
|
||||
},
|
||||
},
|
||||
["."],
|
||||
);
|
||||
|
||||
// 使用統一的封裝管理器建立 .tar（不壓縮）
|
||||
const outputTar = await createJobArchive(outputPath, jobId);
|
||||
|
||||
return Bun.file(outputTar);
|
||||
},
|
||||
{
|
||||
|
|
|
|||
178
src/pages/downloadChunk.tsx
Normal file
178
src/pages/downloadChunk.tsx
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
/**
|
||||
* Contents.CN Chunk 下載 API
|
||||
*
|
||||
* 處理大檔的分段下載
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { join } from "node:path";
|
||||
import { outputDir } from "..";
|
||||
import db from "../db/db";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
shouldUseChunkedDownload,
|
||||
getChunkDownloadInfo,
|
||||
getChunk,
|
||||
createChunkDownloadHeaders,
|
||||
getFileForDirectDownload,
|
||||
createDirectDownloadHeaders,
|
||||
CHUNK_SIZE_BYTES,
|
||||
} from "../transfer";
|
||||
import { statSync, existsSync } from "node:fs";
|
||||
|
||||
export const downloadChunk = new Elysia()
|
||||
.use(userService)
|
||||
/**
|
||||
* 取得檔案下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(filePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* 下載特定 chunk
|
||||
*/
|
||||
.get(
|
||||
"/download/:userId/:jobId/:fileName/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const fileName = sanitize(decodeURIComponent(params.fileName));
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const filePath = `${outputDir}${userId}/${jobId}/${fileName}`;
|
||||
|
||||
const info = getChunkDownloadInfo(filePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "File not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(filePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載資訊
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/info",
|
||||
async ({ params, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
if (!existsSync(archivePath)) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
|
||||
if (!info) {
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
return {
|
||||
...info,
|
||||
use_chunked: shouldUseChunkedDownload(archivePath),
|
||||
};
|
||||
},
|
||||
{ auth: true }
|
||||
)
|
||||
/**
|
||||
* Archive chunk 下載
|
||||
*/
|
||||
.get(
|
||||
"/archive/:jobId/chunk/:chunkIndex",
|
||||
async ({ params, set, user }) => {
|
||||
const userId = user.id;
|
||||
const job = await db
|
||||
.query("SELECT * FROM jobs WHERE user_id = ? AND id = ?")
|
||||
.get(user.id, params.jobId);
|
||||
|
||||
if (!job) {
|
||||
set.status = 404;
|
||||
return { error: "Job not found" };
|
||||
}
|
||||
|
||||
const jobId = decodeURIComponent(params.jobId);
|
||||
const chunkIndex = parseInt(params.chunkIndex, 10);
|
||||
const archivePath = `${outputDir}${userId}/${jobId}/converted_files_${jobId}.tar`;
|
||||
|
||||
const info = getChunkDownloadInfo(archivePath);
|
||||
if (!info) {
|
||||
set.status = 404;
|
||||
return { error: "Archive not found" };
|
||||
}
|
||||
|
||||
const chunkData = await getChunk(archivePath, chunkIndex);
|
||||
if (!chunkData) {
|
||||
set.status = 404;
|
||||
return { error: "Chunk not found" };
|
||||
}
|
||||
|
||||
const headers = createChunkDownloadHeaders(info, chunkIndex, chunkData);
|
||||
|
||||
for (const [key, value] of Object.entries(headers)) {
|
||||
set.headers[key] = value;
|
||||
}
|
||||
|
||||
return new Response(chunkData);
|
||||
},
|
||||
{ auth: true }
|
||||
);
|
||||
88
src/pages/uploadChunk.tsx
Normal file
88
src/pages/uploadChunk.tsx
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
/**
|
||||
* Contents.CN Chunk 上傳 API
|
||||
*
|
||||
* 處理大檔的分段上傳
|
||||
*/
|
||||
|
||||
import { Elysia, t } from "elysia";
|
||||
import { WEBROOT } from "../helpers/env";
|
||||
import { uploadsDir } from "../index";
|
||||
import { userService } from "./user";
|
||||
import sanitize from "sanitize-filename";
|
||||
import {
|
||||
handleChunkUpload,
|
||||
shouldUseChunkedUpload,
|
||||
CHUNK_SIZE_BYTES,
|
||||
calculateChunkCount,
|
||||
} from "../transfer";
|
||||
|
||||
export const uploadChunk = new Elysia().use(userService).post(
|
||||
"/upload-chunk",
|
||||
async ({ body, user, cookie: { jobId } }) => {
|
||||
if (!jobId?.value) {
|
||||
return {
|
||||
success: false,
|
||||
message: "No active job session",
|
||||
};
|
||||
}
|
||||
|
||||
const { upload_id, chunk_index, total_chunks, file_name, total_size, chunk } = body;
|
||||
|
||||
const sanitizedFileName = sanitize(file_name);
|
||||
const userUploadsDir = `${uploadsDir}${user.id}/${jobId.value}/`;
|
||||
|
||||
// 取得 chunk 資料
|
||||
const chunkData = chunk instanceof Blob ? await chunk.arrayBuffer() : chunk;
|
||||
|
||||
const result = await handleChunkUpload(
|
||||
upload_id,
|
||||
parseInt(chunk_index, 10),
|
||||
parseInt(total_chunks, 10),
|
||||
chunkData,
|
||||
sanitizedFileName,
|
||||
parseInt(total_size, 10),
|
||||
user.id,
|
||||
jobId.value,
|
||||
`${uploadsDir}${user.id}/`,
|
||||
userUploadsDir
|
||||
);
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
upload_id: t.String(),
|
||||
chunk_index: t.String(),
|
||||
total_chunks: t.String(),
|
||||
file_name: t.String(),
|
||||
total_size: t.String(),
|
||||
chunk: t.File(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* 取得上傳資訊（用於前端判斷是否需要 chunk 上傳）
|
||||
*/
|
||||
export const uploadInfo = new Elysia().use(userService).post(
|
||||
"/upload-info",
|
||||
async ({ body }) => {
|
||||
const { file_size } = body;
|
||||
const size = parseInt(file_size, 10);
|
||||
|
||||
const useChunked = shouldUseChunkedUpload(size);
|
||||
|
||||
return {
|
||||
use_chunked: useChunked,
|
||||
chunk_size: CHUNK_SIZE_BYTES,
|
||||
total_chunks: useChunked ? calculateChunkCount(size) : 1,
|
||||
};
|
||||
},
|
||||
{
|
||||
body: t.Object({
|
||||
file_size: t.String(),
|
||||
}),
|
||||
auth: true,
|
||||
}
|
||||
);
|
||||
Loading…
Add table
Add a link
Reference in a new issue