From e5ca3645635b90dda3167e5bf4b6e95222afca67 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 21 Jan 2026 12:10:49 +0800 Subject: [PATCH] feat: integrate MinerU document to Markdown converter - Add MinerU converter engine (src/converters/mineru.ts) - Support md-t (table as Markdown) and md-i (table as image) output formats - Input formats: pdf, ppt, pptx, xls, xlsx, doc, docx - Output as ZIP archive containing Markdown and images - Update Dockerfile to install magic-pdf via pipx - Add zip utility for archive creation - Update normalizeFiletype to map md-t/md-i to zip extension - Add comprehensive tests for MinerU converter - Update README with MinerU documentation --- Dockerfile | 2 + README.md | 20 ++- src/converters/main.ts | 5 + src/converters/mineru.ts | 152 +++++++++++++++++++++++ src/helpers/normalizeFiletype.ts | 4 + tests/converters/mineru.test.ts | 204 +++++++++++++++++++++++++++++++ 6 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 src/converters/mineru.ts create mode 100644 tests/converters/mineru.test.ts diff --git a/Dockerfile b/Dockerfile index 1113226..05b0d9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,8 +139,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pipx \ # === 系統工具 === locales \ + zip \ # === 清理 === && pipx install "markitdown[all]" \ + && pipx install magic-pdf \ # 清理 apt cache && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ diff --git a/README.md b/README.md index 86110b5..2ff932d 100644 --- a/README.md +++ b/README.md @@ -165,11 +165,23 @@ docker compose up -d | Pandoc | 文件 | 100+ | | Calibre | 電子書 | 40+ | | Inkscape | 向量圖 | 20+ | +| MinerU | 文件→MD | 2 | 完整列表 → [docs/converters.md](docs/converters.md) --- +## MinerU + +ConvertX 內建 MinerU 轉換引擎，可將文件轉換為 Markdown。 + +- md-t +- md-i + +輸出格式為 ZIP。 + +--- + ## 語言支援 支援 **65 種語言**，包含繁體中文、簡體中文、英文、日文、韓文等。 @@ -214,10 +226,10 @@ docker compose up -d docker compose --profile api up -d ``` -| 服務 | 端口 | 說明 | -| ---------- | ---- | ----------------- | -| Web UI | 3000 | 網頁介面 | -| API Server | 3001 | REST & GraphQL | +| 服務 | 端口 | 說明 | +| ---------- | ---- | -------------- | +| Web UI | 3000 | 網頁介面 | +| API Server | 3001 | REST & GraphQL | ### API 文件 diff --git a/src/converters/main.ts b/src/converters/main.ts index cee3a30..d92d776 100644 --- a/src/converters/main.ts +++ b/src/converters/main.ts @@ -25,6 +25,7 @@ import { convert as convertVtracer, properties as propertiesVtracer } from "./vt import { convert as convertVcf, properties as propertiesVcf } from "./vcf"; import { convert as convertxelatex, properties as propertiesxelatex } from "./xelatex"; import { convert as convertMarkitdown, properties as propertiesMarkitdown } from "./markitdown"; +import { convert as convertMineru, properties as propertiesMineru } from "./mineru"; // This should probably be reconstructed so that the functions are not imported instead the functions hook into this to make the converters more modular @@ -137,6 +138,10 @@ const properties: Record< properties: propertiesMarkitdown, converter: convertMarkitdown, }, + mineru: { + properties: propertiesMineru, + converter: convertMineru, + }, }; function chunks(arr: T[], size: number): T[][] { diff --git a/src/converters/mineru.ts b/src/converters/mineru.ts new file mode 100644 index 0000000..339a0a3 --- /dev/null +++ b/src/converters/mineru.ts @@ -0,0 +1,152 @@ +import { execFile as execFileOriginal } from "node:child_process"; +import { mkdirSync, existsSync, readdirSync, unlinkSync, rmdirSync } from "node:fs"; +import { join, basename, dirname } from "node:path"; +import { ExecFileFn } from "./types"; + +export const properties = { + from: { + document: ["pdf", "ppt", "pptx", "xls", "xlsx", "doc", "docx"], + }, + to: { + document: ["md-t", "md-i"], + }, + outputMode: "archive" as const, +}; + +/** + * Helper function to create a ZIP archive from a directory + */ +function createZipArchive( + sourceDir: string, + outputZip: string, + execFile: ExecFileFn, +): Promise { + return new Promise((resolve, reject) => { + // Use zip command to create archive + execFile( + "zip", + ["-r", outputZip, "."], + (error, stdout, stderr) => { + if (error) { + reject(`zip error: ${error}`); + return; + } + if (stdout) { + console.log(`zip stdout: ${stdout}`); + } + if (stderr) { + console.error(`zip stderr: ${stderr}`); + } + resolve(); + }, + { cwd: sourceDir }, + ); + }); +} + +/** + * Helper function to remove a directory recursively + */ +function removeDir(dirPath: string): void { + if (existsSync(dirPath)) { + const files = readdirSync(dirPath, { withFileTypes: true }); + for (const file of files) { + const filePath = join(dirPath, file.name); + if (file.isDirectory()) { + removeDir(filePath); + } else { + unlinkSync(filePath); + } + } + rmdirSync(dirPath); + } +} + +export async function convert( + filePath: string, + fileType: string, + convertTo: string, + targetPath: string, + options?: unknown, + execFile: ExecFileFn = execFileOriginal, +): Promise { + return new Promise((resolve, reject) => { + // Determine mode based on target format + // md-t = table mode (tables as markdown) + // md-i = image mode (tables as images) + const tableMode = convertTo === "md-t" ? "latex" : "html"; + + // Create a temporary output directory for MinerU + const outputDir = dirname(targetPath); + const inputFileName = basename(filePath, `.${fileType}`); + const mineruOutputDir = join(outputDir, `${inputFileName}_mineru_${convertTo}`); + + // Ensure output directory exists + if (!existsSync(mineruOutputDir)) { + mkdirSync(mineruOutputDir, { recursive: true }); + } + + // Build MinerU command arguments + // MinerU CLI: magic-pdf -p -o -m auto + const args = [ + "-p", + filePath, + "-o", + mineruOutputDir, + "-m", + "auto", + ]; + + // Add table mode option if md-i (render tables as images) + if (convertTo === "md-i") { + args.push("--table-mode", "image"); + } else { + args.push("--table-mode", "markdown"); + } + + execFile("magic-pdf", args, async (error, stdout, stderr) => { + if (error) { + reject(`mineru error: ${error}`); + return; + } + + if (stdout) { + console.log(`mineru stdout: ${stdout}`); + } + + if (stderr) { + console.error(`mineru stderr: ${stderr}`); + } + + try { + // MinerU outputs to a subdirectory, find the actual output + const mineruActualOutput = join(mineruOutputDir, "auto"); + + // Create ZIP archive from the output directory + const zipPath = targetPath.endsWith(".zip") + ? targetPath + : `${targetPath}.zip`; + + // Ensure the parent directory exists + const zipDir = dirname(zipPath); + if (!existsSync(zipDir)) { + mkdirSync(zipDir, { recursive: true }); + } + + // Use the actual MinerU output directory for zipping + const outputToZip = existsSync(mineruActualOutput) + ? mineruActualOutput + : mineruOutputDir; + + await createZipArchive(outputToZip, zipPath, execFile); + + // Clean up the temporary directory + removeDir(mineruOutputDir); + + resolve("Done"); + } catch (zipError) { + reject(`Failed to create ZIP archive: ${zipError}`); + } + }); + }); +} diff --git a/src/helpers/normalizeFiletype.ts b/src/helpers/normalizeFiletype.ts index cf9389a..b0834f8 100644 --- a/src/helpers/normalizeFiletype.ts +++ b/src/helpers/normalizeFiletype.ts @@ -31,6 +31,10 @@ export const normalizeOutputFiletype = (filetype: string): string => { case "markdown_mmd": case "markdown": return "md"; + // MinerU output formats - output as ZIP + case "md-t": + case "md-i": + return "zip"; default: return lowercaseFiletype; } diff --git a/tests/converters/mineru.test.ts b/tests/converters/mineru.test.ts new file mode 100644 index 0000000..990b7f7 --- /dev/null +++ b/tests/converters/mineru.test.ts @@ -0,0 +1,204 @@ +import { test, expect, describe, beforeEach, afterEach, mock } from "bun:test"; +import { convert, properties } from "../../src/converters/mineru"; +import type { ExecFileException } from "node:child_process"; +import { ExecFileFn } from "../../src/converters/types"; +import { mkdirSync, existsSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +// Skip common tests as MinerU has different behavior (archive output) +test.skip("dummy - required to trigger test detection", () => {}); + +describe("MinerU converter properties", () => { + test("should have correct input formats", () => { + expect(properties.from.document).toContain("pdf"); + expect(properties.from.document).toContain("ppt"); + expect(properties.from.document).toContain("pptx"); + expect(properties.from.document).toContain("xls"); + expect(properties.from.document).toContain("xlsx"); + expect(properties.from.document).toContain("doc"); + expect(properties.from.document).toContain("docx"); + }); + + test("should have correct output formats", () => { + expect(properties.to.document).toContain("md-t"); + expect(properties.to.document).toContain("md-i"); + }); + + test("should have archive output mode", () => { + expect(properties.outputMode).toBe("archive"); + }); +}); + +describe("MinerU converter md-t mode", () => { + const testDir = "./test-output-mineru-t"; + + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("should call magic-pdf with markdown table mode for md-t", async () => { + let magicPdfArgs: string[] = []; + + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "magic-pdf") { + magicPdfArgs = args; + // Simulate MinerU creating output + const outputDir = args[3]; // -o argument + if (outputDir && !existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + const autoDir = join(outputDir, "auto"); + if (!existsSync(autoDir)) { + mkdirSync(autoDir, { recursive: true }); + } + writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |"); + callback(null, "MinerU conversion complete", ""); + } else if (cmd === "zip") { + // Simulate zip creation + callback(null, "Archive created", ""); + } + }; + + const targetPath = join(testDir, "output.zip"); + await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); + + expect(magicPdfArgs).toContain("--table-mode"); + expect(magicPdfArgs).toContain("markdown"); + }); +}); + +describe("MinerU converter md-i mode", () => { + const testDir = "./test-output-mineru-i"; + + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("should call magic-pdf with image table mode for md-i", async () => { + let capturedArgs: string[] = []; + + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "magic-pdf") { + capturedArgs = args; + // Simulate MinerU creating output + const outputDir = args[3]; // -o argument + if (outputDir && !existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + const autoDir = join(outputDir, "auto"); + if (!existsSync(autoDir)) { + mkdirSync(autoDir, { recursive: true }); + } + writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)"); + callback(null, "MinerU conversion complete", ""); + } else if (cmd === "zip") { + // Simulate zip creation + callback(null, "Archive created", ""); + } + }; + + const targetPath = join(testDir, "output.zip"); + await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile); + + expect(capturedArgs).toContain("--table-mode"); + expect(capturedArgs).toContain("image"); + }); +}); + +describe("MinerU converter ZIP output", () => { + const testDir = "./test-output-mineru-zip"; + + beforeEach(() => { + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + }); + + afterEach(() => { + if (existsSync(testDir)) { + rmSync(testDir, { recursive: true, force: true }); + } + }); + + test("should create ZIP archive from output", async () => { + let zipCalled = false; + let zipArgs: string[] = []; + + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "magic-pdf") { + // Simulate MinerU creating output + const outputDir = args[3]; // -o argument + if (outputDir && !existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + const autoDir = join(outputDir, "auto"); + if (!existsSync(autoDir)) { + mkdirSync(autoDir, { recursive: true }); + } + writeFileSync(join(autoDir, "output.md"), "# Test content"); + mkdirSync(join(autoDir, "images"), { recursive: true }); + writeFileSync(join(autoDir, "images", "img1.png"), "fake image data"); + callback(null, "MinerU conversion complete", ""); + } else if (cmd === "zip") { + zipCalled = true; + zipArgs = args; + callback(null, "Archive created", ""); + } + }; + + const targetPath = join(testDir, "output.zip"); + const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile); + + expect(result).toBe("Done"); + expect(zipCalled).toBe(true); + expect(zipArgs).toContain("-r"); + }); + + test("should reject on magic-pdf error", async () => { + const mockExecFile: ExecFileFn = ( + cmd: string, + args: string[], + callback: (err: ExecFileException | null, stdout: string, stderr: string) => void, + options?: any, + ) => { + if (cmd === "magic-pdf") { + callback(new Error("MinerU failed") as ExecFileException, "", "Error processing file"); + } + }; + + const targetPath = join(testDir, "output.zip"); + expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile)) + .rejects.toMatch(/mineru error/); + }); +});