Elijah/convertor
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: update MinerU to use tar.gz and mineru CLI

Browse source 
- Change archive format from zip to tar.gz
- Use 'mineru' CLI command instead of 'magic-pdf'
- Install mineru[all] via pipx in Dockerfile
- Remove zip dependency from Dockerfile
- Update normalizeFiletype to output tar.gz extension
- Update all tests for tar.gz output format
- Simplify README MinerU section
This commit is contained in:
Your Name 2026-01-21 12:20:40 +08:00
parent e5ca364563
commit f7ebc084ea
5 changed files with 93 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Dockerfile
View file

@ -139,10 +139,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pipx \
# === 系統工具 ===
locales \
zip \
# === 清理 ===
&& pipx install "markitdown[all]" \
&& pipx install magic-pdf \
&& pipx install "mineru[all]" \
# 清理 apt cache
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \

4
README.md
View file

@ -173,12 +173,12 @@ docker compose up -d
## MinerU
ConvertX 內建 MinerU 轉換引擎，可將文件轉換為 Markdown。
ConvertX 內建文件轉換引擎。
- md-t
- md-i
輸出格式為 ZIP。
輸出格式：tar.gz
---

44
src/converters/mineru.ts
View file

@ -14,32 +14,32 @@ export const properties = {
};
/**
* Helper function to create a ZIP archive from a directory
* Helper function to create a tar.gz archive from a directory
*/
function createZipArchive(
function createTarGzArchive(
sourceDir: string,
outputZip: string,
outputTarGz: string,
execFile: ExecFileFn,
): Promise<void> {
return new Promise((resolve, reject) => {
// Use zip command to create archive
// Use tar command to create gzipped archive
// tar -czf <output.tar.gz> -C <sourceDir> .
execFile(
"zip",
["-r", outputZip, "."],
"tar",
["-czf", outputTarGz, "-C", sourceDir, "."],
(error, stdout, stderr) => {
if (error) {
reject(`zip error: ${error}`);
reject(`tar error: ${error}`);
return;
}
if (stdout) {
console.log(`zip stdout: ${stdout}`);
console.log(`tar stdout: ${stdout}`);
}
if (stderr) {
console.error(`zip stderr: ${stderr}`);
console.error(`tar stderr: ${stderr}`);
}
resolve();
},
{ cwd: sourceDir },
);
});
}
@ -104,7 +104,7 @@ export async function convert(
args.push("--table-mode", "markdown");
}
execFile("magic-pdf", args, async (error, stdout, stderr) => {
execFile("mineru", args, async (error, stdout, stderr) => {
if (error) {
reject(`mineru error: ${error}`);
return;
@ -122,30 +122,30 @@ export async function convert(
// MinerU outputs to a subdirectory, find the actual output
const mineruActualOutput = join(mineruOutputDir, "auto");
// Create ZIP archive from the output directory
const zipPath = targetPath.endsWith(".zip")
// Create tar.gz archive from the output directory
const tarGzPath = targetPath.endsWith(".tar.gz")
? targetPath
: `${targetPath}.zip`;
: `${targetPath}.tar.gz`;
// Ensure the parent directory exists
const zipDir = dirname(zipPath);
if (!existsSync(zipDir)) {
mkdirSync(zipDir, { recursive: true });
const tarGzDir = dirname(tarGzPath);
if (!existsSync(tarGzDir)) {
mkdirSync(tarGzDir, { recursive: true });
}
// Use the actual MinerU output directory for zipping
const outputToZip = existsSync(mineruActualOutput)
// Use the actual MinerU output directory for archiving
const outputToArchive = existsSync(mineruActualOutput)
? mineruActualOutput
: mineruOutputDir;
await createZipArchive(outputToZip, zipPath, execFile);
await createTarGzArchive(outputToArchive, tarGzPath, execFile);
// Clean up the temporary directory
removeDir(mineruOutputDir);
resolve("Done");
} catch (zipError) {
reject(`Failed to create ZIP archive: ${zipError}`);
} catch (tarError) {
reject(`Failed to create tar.gz archive: ${tarError}`);
}
});
});

4
src/helpers/normalizeFiletype.ts
View file

@ -31,10 +31,10 @@ export const normalizeOutputFiletype = (filetype: string): string => {
case "markdown_mmd":
case "markdown":
return "md";
// MinerU output formats - output as ZIP
// MinerU output formats - output as tar.gz
case "md-t":
case "md-i":
return "zip";
return "tar.gz";
default:
return lowercaseFiletype;
}

96
tests/converters/mineru.test.ts
View file

@ -1,4 +1,4 @@
import { test, expect, describe, beforeEach, afterEach, mock } from "bun:test";
import { test, expect, describe, beforeEach, afterEach } from "bun:test";
import { convert, properties } from "../../src/converters/mineru";
import type { ExecFileException } from "node:child_process";
import { ExecFileFn } from "../../src/converters/types";
@ -44,8 +44,8 @@ describe("MinerU converter md-t mode", () => {
}
});
test("should call magic-pdf with markdown table mode for md-t", async () => {
let magicPdfArgs: string[] = [];
test("should call mineru with markdown table mode for md-t", async () => {
let mineruArgs: string[] = [];
const mockExecFile: ExecFileFn = (
cmd: string,
@ -53,8 +53,8 @@ describe("MinerU converter md-t mode", () => {
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "magic-pdf") {
magicPdfArgs = args;
if (cmd === "mineru") {
mineruArgs = args;
// Simulate MinerU creating output
const outputDir = args[3]; // -o argument
if (outputDir && !existsSync(outputDir)) {
@ -66,17 +66,17 @@ describe("MinerU converter md-t mode", () => {
}
writeFileSync(join(autoDir, "output.md"), "# Test\n\n| Col1 | Col2 |\n|---|---|\n| A | B |");
callback(null, "MinerU conversion complete", "");
} else if (cmd === "zip") {
// Simulate zip creation
} else if (cmd === "tar") {
// Simulate tar.gz creation
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "output.zip");
const targetPath = join(testDir, "output.tar.gz");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(magicPdfArgs).toContain("--table-mode");
expect(magicPdfArgs).toContain("markdown");
expect(mineruArgs).toContain("--table-mode");
expect(mineruArgs).toContain("markdown");
});
});
@ -95,7 +95,7 @@ describe("MinerU converter md-i mode", () => {
}
});
test("should call magic-pdf with image table mode for md-i", async () => {
test("should call mineru with image table mode for md-i", async () => {
let capturedArgs: string[] = [];
const mockExecFile: ExecFileFn = (
@ -104,7 +104,7 @@ describe("MinerU converter md-i mode", () => {
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "magic-pdf") {
if (cmd === "mineru") {
capturedArgs = args;
// Simulate MinerU creating output
const outputDir = args[3]; // -o argument
@ -117,13 +117,13 @@ describe("MinerU converter md-i mode", () => {
}
writeFileSync(join(autoDir, "output.md"), "# Test\n\n![Table](images/table_1.png)");
callback(null, "MinerU conversion complete", "");
} else if (cmd === "zip") {
// Simulate zip creation
} else if (cmd === "tar") {
// Simulate tar.gz creation
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "output.zip");
const targetPath = join(testDir, "output.tar.gz");
await convert("test.pdf", "pdf", "md-i", targetPath, undefined, mockExecFile);
expect(capturedArgs).toContain("--table-mode");
@ -131,8 +131,8 @@ describe("MinerU converter md-i mode", () => {
});
});
describe("MinerU converter ZIP output", () => {
const testDir = "./test-output-mineru-zip";
describe("MinerU converter tar.gz output", () => {
const testDir = "./test-output-mineru-targz";
beforeEach(() => {
if (!existsSync(testDir)) {
@ -146,9 +146,9 @@ describe("MinerU converter ZIP output", () => {
}
});
test("should create ZIP archive from output", async () => {
let zipCalled = false;
let zipArgs: string[] = [];
test("should create tar.gz archive from output", async () => {
let tarCalled = false;
let tarArgs: string[] = [];
const mockExecFile: ExecFileFn = (
cmd: string,
@ -156,7 +156,7 @@ describe("MinerU converter ZIP output", () => {
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "magic-pdf") {
if (cmd === "mineru") {
// Simulate MinerU creating output
const outputDir = args[3]; // -o argument
if (outputDir && !existsSync(outputDir)) {
@ -170,35 +170,71 @@ describe("MinerU converter ZIP output", () => {
mkdirSync(join(autoDir, "images"), { recursive: true });
writeFileSync(join(autoDir, "images", "img1.png"), "fake image data");
callback(null, "MinerU conversion complete", "");
} else if (cmd === "zip") {
zipCalled = true;
zipArgs = args;
} else if (cmd === "tar") {
tarCalled = true;
tarArgs = args;
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "output.zip");
const targetPath = join(testDir, "output.tar.gz");
const result = await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
expect(result).toBe("Done");
expect(zipCalled).toBe(true);
expect(zipArgs).toContain("-r");
expect(tarCalled).toBe(true);
expect(tarArgs).toContain("-czf");
});
test("should reject on magic-pdf error", async () => {
test("should reject on mineru error", async () => {
const mockExecFile: ExecFileFn = (
cmd: string,
args: string[],
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "magic-pdf") {
if (cmd === "mineru") {
callback(new Error("MinerU failed") as ExecFileException, "", "Error processing file");
}
};
const targetPath = join(testDir, "output.zip");
const targetPath = join(testDir, "output.tar.gz");
expect(convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile))
.rejects.toMatch(/mineru error/);
});
test("should use correct tar arguments for compression", async () => {
let tarArgs: string[] = [];
const mockExecFile: ExecFileFn = (
cmd: string,
args: string[],
callback: (err: ExecFileException | null, stdout: string, stderr: string) => void,
options?: any,
) => {
if (cmd === "mineru") {
const outputDir = args[3];
if (outputDir && !existsSync(outputDir)) {
mkdirSync(outputDir, { recursive: true });
}
const autoDir = join(outputDir, "auto");
if (!existsSync(autoDir)) {
mkdirSync(autoDir, { recursive: true });
}
writeFileSync(join(autoDir, "output.md"), "# Test");
callback(null, "Done", "");
} else if (cmd === "tar") {
tarArgs = args;
callback(null, "Archive created", "");
}
};
const targetPath = join(testDir, "test_MINERU_md-t.tar.gz");
await convert("test.pdf", "pdf", "md-t", targetPath, undefined, mockExecFile);
// Verify tar is called with correct compression flags
expect(tarArgs[0]).toBe("-czf");
expect(tarArgs[1]).toContain(".tar.gz");
expect(tarArgs[2]).toBe("-C");
expect(tarArgs[4]).toBe(".");
});
});