diff --git a/Dockerfile b/Dockerfile index 05b0d9c..511edda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,10 +139,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pipx \ # === 系統工具 === locales \ - zip \ # === 清理 === && pipx install "markitdown[all]" \ - && pipx install magic-pdf \ + && pipx install "mineru[all]" \ # 清理 apt cache && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ diff --git a/README.md b/README.md index 2ff932d..3d20689 100644 --- a/README.md +++ b/README.md @@ -173,12 +173,12 @@ docker compose up -d ## MinerU -ConvertX 內建 MinerU 轉換引擎，可將文件轉換為 Markdown。 +ConvertX 內建文件轉換引擎。 - md-t - md-i -輸出格式為 ZIP。 +輸出格式：tar.gz --- diff --git a/src/converters/mineru.ts b/src/converters/mineru.ts index 339a0a3..ce81d8f 100644 --- a/src/converters/mineru.ts +++ b/src/converters/mineru.ts @@ -14,32 +14,32 @@ export const properties = { }; /** - * Helper function to create a ZIP archive from a directory + * Helper function to create a tar.gz archive from a directory */ -function createZipArchive( +function createTarGzArchive( sourceDir: string, - outputZip: string, + outputTarGz: string, execFile: ExecFileFn, ): Promise { return new Promise((resolve, reject) => { - // Use zip command to create archive + // Use tar command to create gzipped archive + // tar -czf -C . execFile( - "zip", - ["-r", outputZip, "."], + "tar", + ["-czf", outputTarGz, "-C", sourceDir, "."], (error, stdout, stderr) => { if (error) { - reject(`zip error: ${error}`); + reject(`tar error: ${error}`); return; } if (stdout) { - console.log(`zip stdout: ${stdout}`); + console.log(`tar stdout: ${stdout}`); } if (stderr) { - console.error(`zip stderr: ${stderr}`); + console.error(`tar stderr: ${stderr}`); } resolve(); }, - { cwd: sourceDir }, ); }); } @@ -104,7 +104,7 @@ export async function convert( args.push("--table-mode", "markdown"); } - execFile("magic-pdf", args, async (error, stdout, stderr) => { + execFile("mineru", args, async (error, stdout, stderr) => { if (error) { reject(`mineru error: ${error}`); return; @@ -122,30 +122,30 @@ export async function convert( // MinerU outputs to a subdirectory, find the actual output const mineruActualOutput = join(mineruOutputDir, "auto"); - // Create ZIP archive from the output directory - const zipPath = targetPath.endsWith(".zip") + // Create tar.gz archive from the output directory + const tarGzPath = targetPath.endsWith(".tar.gz") ? targetPath - : `${targetPath}.zip`; + : `${targetPath}.tar.gz`; // Ensure the parent directory exists - const zipDir = dirname(zipPath); - if (!existsSync(zipDir)) { - mkdirSync(zipDir, { recursive: true }); + const tarGzDir = dirname(tarGzPath); + if (!existsSync(tarGzDir)) { + mkdirSync(tarGzDir, { recursive: true }); } - // Use the actual MinerU output directory for zipping - const outputToZip = existsSync(mineruActualOutput) + // Use the actual MinerU output directory for archiving + const outputToArchive = existsSync(mineruActualOutput) ? mineruActualOutput : mineruOutputDir; - await createZipArchive(outputToZip, zipPath, execFile); + await createTarGzArchive(outputToArchive, tarGzPath, execFile); // Clean up the temporary directory removeDir(mineruOutputDir); resolve("Done"); - } catch (zipError) { - reject(`Failed to create ZIP archive: ${zipError}`); + } catch (tarError) { + reject(`Failed to create tar.gz archive: ${tarError}`); } }); }); diff --git a/src/helpers/normalizeFiletype.ts b/src/helpers/normalizeFiletype.ts index b0834f8..d719a71 100644 --- a/src/helpers/normalizeFiletype.ts +++ b/src/helpers/normalizeFiletype.ts @@ -31,10 +31,10 @@ export const normalizeOutputFiletype = (filetype: string): string => { case "markdown_mmd": case "markdown": return "md"; - // MinerU output formats - output as ZIP + // MinerU output formats - output as tar.gz case "md-t": case "md-i": - return "zip"; + return "tar.gz"; default: return lowercaseFiletype; } diff --git a/tests/converters/mineru.test.ts b/tests/converters/mineru.test.ts index 990b7f7..d2226f0 100644 --- a/tests/converters/mineru.test.ts +++ b/tests/converters/mineru.test.ts @@ -1,4 +1,4 @@ -import { test, expect, describe, beforeEach, afterEach, mock } from "bun:test"; +import { test, expect, describe, beforeEach, afterEach } from "bun:test"; import { convert, properties } from "../../src/converters/mineru"; import type { ExecFileException } from "node:child_process"; import { ExecFileFn } from "../../src/converters/types"; @@ -44,8 +44,8 @@ describe("MinerU converter md-t mode", () => { } }); - test("should call magic-pdf with markdown table mode for md-t", async () => { - let magicPdfArgs: string[] = []; + test("should call mineru with markdown table mode for md-t", async () => { + let mineruArgs: string[] = []; const mockExecFile: ExecFileFn = ( cmd: string, @@ -53,8 +53,8 @@ describe("MinerU converter md-t mode", () => { callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, options?: any, ) => { - if (cmd === "magic-pdf") { - magicPdfArgs = args; + if (cmd === "mineru") { + mineruArgs = args; // Simulate MinerU creating output const outputDir = args[3]; // -o argument if (outputDir && !existsSync(outputDir)) { @@ -66,17 +66,17 @@ describe("MinerU converter md-t mode", () => { } writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |"); callback(null, "MinerU conversion complete", ""); - } else if (cmd === "zip") { - // Simulate zip creation + } else if (cmd === "tar") { + // Simulate tar.gz creation callback(null, "Archive created", ""); } }; - const targetPath = join(testDir, "output.zip"); + const targetPath = join(testDir, "output.tar.gz"); await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); - expect(magicPdfArgs).toContain("--table-mode"); - expect(magicPdfArgs).toContain("markdown"); + expect(mineruArgs).toContain("--table-mode"); + expect(mineruArgs).toContain("markdown"); }); }); @@ -95,7 +95,7 @@ describe("MinerU converter md-i mode", () => { } }); - test("should call magic-pdf with image table mode for md-i", async () => { + test("should call mineru with image table mode for md-i", async () => { let capturedArgs: string[] = []; const mockExecFile: ExecFileFn = ( @@ -104,7 +104,7 @@ describe("MinerU converter md-i mode", () => { callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, options?: any, ) => { - if (cmd === "magic-pdf") { + if (cmd === "mineru") { capturedArgs = args; // Simulate MinerU creating output const outputDir = args[3]; // -o argument @@ -117,13 +117,13 @@ describe("MinerU converter md-i mode", () => { } writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)"); callback(null, "MinerU conversion complete", ""); - } else if (cmd === "zip") { - // Simulate zip creation + } else if (cmd === "tar") { + // Simulate tar.gz creation callback(null, "Archive created", ""); } }; - const targetPath = join(testDir, "output.zip"); + const targetPath = join(testDir, "output.tar.gz"); await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile); expect(capturedArgs).toContain("--table-mode"); @@ -131,8 +131,8 @@ describe("MinerU converter md-i mode", () => { }); }); -describe("MinerU converter ZIP output", () => { - const testDir = "./test-output-mineru-zip"; +describe("MinerU converter tar.gz output", () => { + const testDir = "./test-output-mineru-targz"; beforeEach(() => { if (!existsSync(testDir)) { @@ -146,9 +146,9 @@ describe("MinerU converter ZIP output", () => { } }); - test("should create ZIP archive from output", async () => { - let zipCalled = false; - let zipArgs: string[] = []; + test("should create tar.gz archive from output", async () => { + let tarCalled = false; + let tarArgs: string[] = []; const mockExecFile: ExecFileFn = ( cmd: string, @@ -156,7 +156,7 @@ describe("MinerU converter ZIP output", () => { callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, options?: any, ) => { - if (cmd === "magic-pdf") { + if (cmd === "mineru") { // Simulate MinerU creating output const outputDir = args[3]; // -o argument if (outputDir && !existsSync(outputDir)) { @@ -170,35 +170,71 @@ describe("MinerU converter ZIP output", () => { mkdirSync(join(autoDir, "images"), { recursive: true }); writeFileSync(join(autoDir, "images", "img1.png"), "fake image data"); callback(null, "MinerU conversion complete", ""); - } else if (cmd === "zip") { - zipCalled = true; - zipArgs = args; + } else if (cmd === "tar") { + tarCalled = true; + tarArgs = args; callback(null, "Archive created", ""); } }; - const targetPath = join(testDir, "output.zip"); + const targetPath = join(testDir, "output.tar.gz"); const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); expect(result).toBe("Done"); - expect(zipCalled).toBe(true); - expect(zipArgs).toContain("-r"); + expect(tarCalled).toBe(true); + expect(tarArgs).toContain("-czf"); }); - test("should reject on magic-pdf error", async () => { + test("should reject on mineru error", async () => { const mockExecFile: ExecFileFn = ( cmd: string, args: string[], callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, options?: any, ) => { - if (cmd === "magic-pdf") { + if (cmd === "mineru") { callback(new Error("MinerU failed") as ExecFileException, "", "Error processing file"); } }; - const targetPath = join(testDir, "output.zip"); + const targetPath = join(testDir, "output.tar.gz"); expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile)) .rejects.toMatch(/mineru error/); }); + + test("should use correct tar arguments for compression", async () => { + let tarArgs: string[] = []; + + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "mineru") { + const outputDir = args[3]; + if (outputDir && !existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + const autoDir = join(outputDir, "auto"); + if (!existsSync(autoDir)) { + mkdirSync(autoDir, { recursive: true }); + } + writeFileSync(join(autoDir, "output.md"), "# Test"); + callback(null, "Done", ""); + } else if (cmd === "tar") { + tarArgs = args; + callback(null, "Archive created", ""); + } + }; + + const targetPath = join(testDir, "test_MINERU_md-t.tar.gz"); + await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); + + // Verify tar is called with correct compression flags + expect(tarArgs[0]).toBe("-czf"); + expect(tarArgs[1]).toContain(".tar.gz"); + expect(tarArgs[2]).toBe("-C"); + expect(tarArgs[4]).toBe("."); + }); });