فهرست منبع

fix: strip markdown metadata from length metrics

Ma 7 ماه پیش
والد
کامیت
9be31562d4
2فایلهای تغییر یافته به همراه55 افزوده شده و 1 حذف شده
  1. 14 0
      packages/core/src/__tests__/length-metrics.test.ts
  2. 41 1
      packages/core/src/utils/length-metrics.ts

+ 14 - 0
packages/core/src/__tests__/length-metrics.test.ts

@@ -16,6 +16,20 @@ describe("length metrics", () => {
     expect(countChapterLength("He looked at the sky.", "en_words")).toBe(5);
     expect(countChapterLength("He looked at the sky.", "en_words")).toBe(5);
   });
   });
 
 
+  it("counts prose only for markdown-shaped Chinese chapters", () => {
+    const markdownChapter = [
+      "---",
+      "title: 第1章 归来",
+      "---",
+      "",
+      "# 第1章 归来",
+      "",
+      "陈风抬头看天。",
+    ].join("\n");
+
+    expect(countChapterLength(markdownChapter, "zh_chars")).toBe("陈风抬头看天。".length);
+  });
+
   it("builds a conservative length spec for Chinese chapters", () => {
   it("builds a conservative length spec for Chinese chapters", () => {
     const spec = buildLengthSpec(2200, "zh");
     const spec = buildLengthSpec(2200, "zh");
 
 

+ 41 - 1
packages/core/src/utils/length-metrics.ts

@@ -9,7 +9,7 @@ export function countChapterLength(
   content: string,
   content: string,
   countingMode: LengthCountingMode,
   countingMode: LengthCountingMode,
 ): number {
 ): number {
-  const normalized = content.replace(/\r\n/g, "\n");
+  const normalized = stripMarkdownMetadata(content);
 
 
   if (countingMode === "en_words") {
   if (countingMode === "en_words") {
     const words = normalized.match(/[A-Za-z0-9]+(?:'[A-Za-z0-9]+)?/g);
     const words = normalized.match(/[A-Za-z0-9]+(?:'[A-Za-z0-9]+)?/g);
@@ -61,3 +61,43 @@ export function chooseNormalizeMode(
   if (count > spec.softMax) return "compress";
   if (count > spec.softMax) return "compress";
   return "none";
   return "none";
 }
 }
+
+function stripMarkdownMetadata(content: string): string {
+  const lines = content.replace(/\r\n/g, "\n").replace(/^\uFEFF/, "").split("\n");
+  const proseLines: string[] = [];
+  let index = 0;
+
+  if (lines[index]?.trim() === "---") {
+    index += 1;
+    while (index < lines.length && lines[index]?.trim() !== "---") {
+      index += 1;
+    }
+    if (index < lines.length) {
+      index += 1;
+    }
+  }
+
+  let inFence = false;
+  for (; index < lines.length; index += 1) {
+    const line = lines[index] ?? "";
+    const trimmed = line.trim();
+
+    if (/^(```|~~~)/.test(trimmed)) {
+      inFence = !inFence;
+      continue;
+    }
+    if (inFence) {
+      continue;
+    }
+    if (/^#{1,6}\s+/.test(trimmed)) {
+      continue;
+    }
+    if (trimmed === "---" || trimmed === "...") {
+      continue;
+    }
+
+    proseLines.push(line);
+  }
+
+  return proseLines.join("\n");
+}