Merge f161f68240 into 393441faa1

Dockerfile

@@ -73,9 +73,11 @@ RUN apt-get update && apt-get install -y \
   resvg \
   texlive \
   texlive-fonts-recommended \
+  texlive-lang-chinese \
   texlive-latex-extra \
   texlive-latex-recommended \
   texlive-xetex \
+  fonts-noto-cjk \
   python3 \
   python3-pip \
   pipx \

src/converters/pandoc.ts

@@ -1,4 +1,5 @@
 import { execFile as execFileOriginal } from "node:child_process";
+import { readFileSync } from "node:fs";
 import { ExecFileFn } from "./types";
 
 export const properties = {
@@ -120,6 +121,52 @@ export const properties = {
   },
 };
 
+/**
+ * Input formats whose files are binary or compressed (ZIP-based) and cannot
+ * be reliably scanned for CJK characters by reading raw bytes. For these
+ * formats, a CJK font is always passed to avoid missing glyphs.
+ */
+const binaryFormats = new Set(["docx", "epub", "odt", "pptx"]);
+
+/**
+ * Detects CJK characters in a file and returns the appropriate Noto Sans CJK
+ * font variant for the detected language. Returns null for non-CJK content
+ * so that the CJK font argument is omitted entirely, keeping non-CJK
+ * conversions working in environments without CJK fonts installed.
+ *
+ * Detection order matters: Japanese Kana is checked first (most specific),
+ * then Korean Hangul, then CJK Unified Ideographs (Chinese). This is because
+ * Japanese text also uses Kanji (shared with Chinese), but the presence of
+ * Kana uniquely identifies Japanese content.
+ */
+function detectCJKFont(filePath: string): string | null {
+  let content: string;
+  try {
+    content = readFileSync(filePath, "utf-8");
+  } catch {
+    // If the file can't be read, don't add a CJK font to avoid breaking
+    // conversions. Pandoc will still process the file independently.
+    return null;
+  }
+
+  // Japanese: Hiragana (U+3040-309F) or Katakana (U+30A0-30FF)
+  if (/[\u3040-\u309f\u30a0-\u30ff]/.test(content)) {
+    return "Noto Sans CJK JP";
+  }
+
+  // Korean: Hangul Syllables (U+AC00-D7AF)
+  if (/[\uac00-\ud7af]/.test(content)) {
+    return "Noto Sans CJK KR";
+  }
+
+  // Chinese: CJK Unified Ideographs (U+4E00-9FFF)
+  if (/[\u4e00-\u9fff]/.test(content)) {
+    return "Noto Sans CJK SC";
+  }
+
+  return null;
+}
+
 export function convert(
   filePath: string,
   fileType: string,
@@ -136,6 +183,21 @@ export function convert(
 
   if (xelatex.includes(convertTo)) {
     args.push("--pdf-engine=xelatex");
+    // For binary/compressed formats (docx, epub, etc.), the raw file bytes
+    // cannot be scanned for CJK characters, so always pass a CJK font.
+    // For text-based formats, detect CJK characters to select the correct
+    // locale-specific font variant (JP/KR/SC) and to avoid adding a CJK
+    // font when the document contains none, keeping non-CJK conversions
+    // working in environments without CJK fonts installed.
+    let cjkFont: string | null;
+    if (binaryFormats.has(fileType.toLowerCase())) {
+      cjkFont = "Noto Sans CJK SC";
+    } else {
+      cjkFont = detectCJKFont(filePath);
+    }
+    if (cjkFont) {
+      args.push("-V", `CJKmainfont=${cjkFont}`);
+    }
   }
 
   args.push(filePath);

tests/converters/pandoc.test.ts

@@ -1,21 +1,51 @@
-import { beforeEach, expect, test, describe } from "bun:test";
+import { beforeEach, expect, test, describe, afterEach } from "bun:test";
 import { convert } from "../../src/converters/pandoc";
 import type { ExecFileFn } from "../../src/converters/types";
+import { writeFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
 
 describe("convert", () => {
   let mockExecFile: ExecFileFn;
+  let tempFiles: string[] = [];
+
+  afterEach(() => {
+    for (const f of tempFiles) {
+      try {
+        unlinkSync(f);
+      } catch {
+        /* ignore */
+      }
+    }
+    tempFiles = [];
+  });
+
+  function makeTempFile(content: string, ext = "md"): string {
+    const p = join(
+      tmpdir(),
+      `pandoc-test-${Date.now()}-${Math.random().toString(36).slice(2)}.${ext}`,
+    );
+    writeFileSync(p, content, "utf-8");
+    tempFiles.push(p);
+    return p;
+  }
+
+  function assertCJKFontArg(args: string[], expectedFont: string) {
+    const idx = args.indexOf(`CJKmainfont=${expectedFont}`);
+    expect(idx).toBeGreaterThan(-1);
+    expect(args[idx - 1]).toBe("-V");
+  }
 
   beforeEach(() => {
     mockExecFile = (cmd, args, callback) => callback(null, "output-data", "");
   });
 
-  test("should call pandoc with correct arguments (normal)", async () => {
+  test("should call pandoc with correct arguments", async () => {
     let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
     mockExecFile = (cmd, args, callback) => {
       calledArgs = [cmd, args, callback];
       callback(null, "output-data", "");
     };
-
     const result = await convert(
       "input.md",
       "markdown",
@@ -24,7 +54,6 @@ describe("convert", () => {
       undefined,
       mockExecFile,
     );
-
     expect(calledArgs[0]).toBe("pandoc");
     expect(calledArgs[1]).toEqual([
       "input.md",
@@ -38,29 +67,127 @@ describe("convert", () => {
     expect(result).toBe("Done");
   });
 
-  test("should add xelatex argument for pdf/latex", async () => {
+  test("should add xelatex argument for pdf", async () => {
     let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
     mockExecFile = (cmd, args, callback) => {
       calledArgs = [cmd, args, callback];
       callback(null, "output-data", "");
     };
-
-    await convert("input.md", "markdown", "pdf", "output.pdf", undefined, mockExecFile);
-
+    const filePath = makeTempFile("# Hello World\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
     expect(calledArgs[1][0]).toBe("--pdf-engine=xelatex");
-    expect(calledArgs[1]).toContain("input.md");
-    expect(calledArgs[1]).toContain("-f");
-    expect(calledArgs[1]).toContain("markdown");
-    expect(calledArgs[1]).toContain("-t");
-    expect(calledArgs[1]).toContain("pdf");
-    expect(calledArgs[1]).toContain("-o");
-    expect(calledArgs[1]).toContain("output.pdf");
+    expect(calledArgs[1]).toContain(filePath);
   });
 
+  test("should not add CJK mainfont for non-CJK content", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# Hello World\nThis is English text.\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    expect(calledArgs[1].some((arg) => arg.startsWith("CJKmainfont"))).toBe(false);
+  });
+
+  test("should add CJKmainfont=SC for Chinese content with -V adjacency", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# 中文标题\n这是一段中文内容。\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    assertCJKFontArg(calledArgs[1], "Noto Sans CJK SC");
+  });
+
+  test("should add CJKmainfont=JP for Japanese content with -V adjacency", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# 日本語タイトル\nこれは日本語の内容です。\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    assertCJKFontArg(calledArgs[1], "Noto Sans CJK JP");
+  });
+
+  test("should add CJKmainfont=KR for Korean content with -V adjacency", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# 한국어 제목\n이것은 한국어 내용입니다.\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    assertCJKFontArg(calledArgs[1], "Noto Sans CJK KR");
+  });
+
+  test("should prefer Japanese font when both Kanji and Kana present", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# 日本語のテスト\nこれは漢字とひらがなの文章です。\n");
+    await convert(filePath, "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    assertCJKFontArg(calledArgs[1], "Noto Sans CJK JP");
+    expect(calledArgs[1].some((arg) => arg === "CJKmainfont=Noto Sans CJK SC")).toBe(false);
+  });
+
+  test("should not add CJK mainfont for latex when content is non-CJK", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# English Title\nSome text.\n");
+    await convert(filePath, "markdown", "latex", "output.tex", undefined, mockExecFile);
+    expect(calledArgs[1][0]).toBe("--pdf-engine=xelatex");
+    expect(calledArgs[1].some((arg) => arg.startsWith("CJKmainfont"))).toBe(false);
+  });
+
+  test("should not add CJK mainfont for non-pdf/latex output", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    const filePath = makeTempFile("# 中文标题\n");
+    await convert(filePath, "markdown", "html", "output.html", undefined, mockExecFile);
+    expect(calledArgs[1].some((arg) => arg.startsWith("CJKmainfont"))).toBe(false);
+  });
+
+  test("should handle unreadable file gracefully without CJK font", async () => {
+    let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+    mockExecFile = (cmd, args, callback) => {
+      calledArgs = [cmd, args, callback];
+      callback(null, "output-data", "");
+    };
+    await convert("/nonexistent/file.md", "markdown", "pdf", "output.pdf", undefined, mockExecFile);
+    expect(calledArgs[1][0]).toBe("--pdf-engine=xelatex");
+    expect(calledArgs[1].some((arg) => arg.startsWith("CJKmainfont"))).toBe(false);
+  });
+
+  test.each(["docx", "epub", "odt"])(
+    "should always add CJK font for %s binary format with -V adjacency",
+    async (format: string) => {
+      let calledArgs: Parameters<ExecFileFn> = ["", [], () => {}];
+      mockExecFile = (cmd, args, callback) => {
+        calledArgs = [cmd, args, callback];
+        callback(null, "output-data", "");
+      };
+      const filePath = makeTempFile("binary placeholder content", format);
+      await convert(filePath, format, "pdf", "output.pdf", undefined, mockExecFile);
+      assertCJKFontArg(calledArgs[1], "Noto Sans CJK SC");
+    },
+  );
+
   test("should reject if execFile returns an error", async () => {
+    const filePath = makeTempFile("# Test\n");
     mockExecFile = (cmd, args, callback) => callback(new Error("fail"), "", "");
     await expect(
-      convert("input.md", "markdown", "html", "output.html", undefined, mockExecFile),
+      convert(filePath, "markdown", "html", "output.html", undefined, mockExecFile),
     ).rejects.toMatch(/error: Error: fail/);
   });
 });