diff --git a/tests/test_cjk_typography.py b/tests/test_cjk_typography.py new file mode 100644 index 0000000..5a2bcc2 --- /dev/null +++ b/tests/test_cjk_typography.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Guard the book's two Chinese-typography conventions. + +Neither rule is enforced anywhere else: ``check_project_rules.py`` looks at +fences and links, ``check_emphasis.py`` at bold markers, and the workspace +``format_checker.py`` at quotes, fence languages and heading spacing. Intra-line +typography had no gate at all, and it drifted exactly as far as you would +expect -- one sibling book reached 45% unspaced CJK/Latin boundaries and another +reached a 50/50 split on parentheses, both mixed *within single files*. A +one-off sweep normalised all fourteen books; this file is what stops the next +paragraph from drifting back. + +RULE 1 -- a CJK ideograph may not sit directly against ``[A-Za-z0-9]``. + ``Harness的定义`` / ``第4章`` are violations; ``Harness 的定义`` is not. + +RULE 2 -- a parenthesis whose content contains any CJK must be full-width. + ``(用户命名空间映射)`` is a violation; ``(用户命名空间映射)`` is not. + Latin-only parentheses are NOT touched: ``新建一个层 (Layer)`` and the + ``项目(Agent)的经验`` hugging style both stay legal, because their + content has no CJK. + +Deliberately NOT checked, because the whole cluster agrees on the opposite: +Chinese punctuation against Latin (``(如 UI、监控服务)``) needs no space -- +full-width punctuation already carries its own visual padding. + +Everything a reader never sees as prose is excluded before the scan: fenced code +blocks and display-math blocks are skipped whole, and inline code, math, link +and image targets, HTML and bare URLs are replaced with Private Use Area +placeholders -- characters that can never themselves be mistaken for a letter, a +digit or an ideograph. Every mask pattern excludes that range in turn, because a +greedy ``\\S+`` will otherwise swallow the placeholder its predecessor just left +behind. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKIP_DIRS = {".git", "_book", "node_modules", ".obsidian", "output", ".agent", "_site"} + +PUA_FIRST = 0xE000 +PUA = "\\ue000-\\uf8ff" +CJK = r"一-鿿㐀-䶿" +LATIN = r"A-Za-z0-9" + +FENCE = re.compile(r"^\s*(?:```|~~~)") +MATH_BLOCK = re.compile(r"^\s*\$\$\s*$") +LINK_DEF = re.compile(r"^\s*\[[^\]]+\]:\s*\S+") + +MASKS = ( + re.compile(r""), + re.compile(r"``[^`]+``"), + re.compile(r"`[^`]*`"), + re.compile(r"\$\$[^$]*\$\$"), + # only treat $...$ as math when it actually contains math syntax, so that a + # pair of prices ("$25/$125 per M tokens") is not read as one formula + re.compile(r"\$(?=[^$\n]*[\\^_{])[^$\n]*\$"), + re.compile(r"!?\]\([^)\s]*(?:\s+\"[^\"]*\")?\)"), + # a real tag, not "| <1% | - | <5 | 各类GPU | >400ms |" + re.compile(r"]*)?/?>"), + re.compile("(? str: + counter = [0] + + def repl(_match: "re.Match[str]") -> str: + counter[0] += 1 + return chr(PUA_FIRST + counter[0] - 1) + + for pattern in MASKS: + line = pattern.sub(repl, line) + return line + + +def prose_lines(text: str): + lines = text.split("\n") + in_front = bool(lines) and lines[0].strip() == "---" + in_fence = False + in_math = False + for number, line in enumerate(lines, start=1): + if in_front: + if number > 1 and line.strip() == "---": + in_front = False + continue + if FENCE.match(line): + in_fence = not in_fence + continue + if in_fence: + continue + if MATH_BLOCK.match(line): + in_math = not in_math + continue + if in_math or LINK_DEF.match(line): + continue + yield number, line + + +def markdown_files(root: Path = ROOT): + for path in sorted(root.rglob("*.md")): + if SKIP_DIRS.isdisjoint(path.relative_to(root).parts): + yield path + + +def unspaced(text: str): + """RULE 1 violations.""" + for number, line in prose_lines(text): + for match in ADJACENT.finditer(mask(line)): + yield number, match.group(0), line.strip()[:100] + + +def half_width_parens(text: str): + """RULE 2 violations.""" + for number, line in prose_lines(text): + for match in HALF_PAREN.finditer(mask(line)): + if re.search(f"[{CJK}]", match.group(0)): + yield number, match.group(0)[:40], line.strip()[:100] + + +class CjkTypographyTests(unittest.TestCase): + def test_cjk_and_latin_stay_apart(self) -> None: + found = [ + f"{path.relative_to(ROOT)}:{number}: {hit!r} in {snippet}" + for path in markdown_files() + for number, hit, snippet in unspaced(path.read_text(encoding="utf-8")) + ] + self.assertEqual( + [], found, "缺少中文与拉丁字符之间的半角空格:\n" + "\n".join(found[:40]) + ) + + def test_parens_holding_chinese_are_full_width(self) -> None: + found = [ + f"{path.relative_to(ROOT)}:{number}: {hit!r} in {snippet}" + for path in markdown_files() + for number, hit, snippet in half_width_parens(path.read_text(encoding="utf-8")) + ] + self.assertEqual( + [], found, "含中文的括号应使用全角:\n" + "\n".join(found[:40]) + ) + + def test_both_rules_can_actually_fail(self) -> None: + """A check that cannot fail is not evidence.""" + self.assertEqual([], list(unspaced("使用 Harness 的记忆子系统。\n"))) + self.assertEqual( + [(1, "s的", "使用 Harness的记忆子系统。")], + list(unspaced("使用 Harness的记忆子系统。\n")), + ) + self.assertEqual([(1, "第4", "第4 章")], list(unspaced("第4 章\n"))) + self.assertEqual([], list(half_width_parens("启用(用户命名空间映射)。\n"))) + self.assertEqual( + [(1, "(用户命名空间映射)", "启用 (用户命名空间映射)。")], + list(half_width_parens("启用 (用户命名空间映射)。\n")), + ) + + def test_latin_only_parens_are_left_alone(self) -> None: + for sample in ( + "在基础镜像之上新建一个层 (Layer)。", + "多个生产级智能体项目(Agent)的经验表明。", + "定义一组相关联的应用容器为一个项目 (project)。", + ): + self.assertEqual([], list(half_width_parens(sample + "\n")), sample) + + def test_non_prose_regions_are_excluded(self) -> None: + for sample in ( + "见 `mini_harness/核心.py` 说明。", + "参考 [第 4 章的运行时](04_runtime/README.md)。", + "地址是 https://example.com/中文path 这一条。", + '第4章示意图 的属性不算正文。', + "见 [文档](a.md) https://e.com/x 结束。", + "公式 $$L = (预测 - 真实)^2$$ 属于数学。", + "记 $a^*$ 与 $O_t$ 为最优解。", + ): + self.assertEqual([], list(unspaced(sample + "\n")), sample) + self.assertEqual([], list(half_width_parens(sample + "\n")), sample) + + def test_prices_are_not_mistaken_for_math(self) -> None: + """A pair of '$' around prose must not mask the prose between them.""" + line = "定价 $25/$125 per M tokens,另有 $1 亿额度给 Glasswing计划。\n" + self.assertEqual([(1, "g计", line.strip())], list(unspaced(line))) + + def test_html_text_content_is_still_prose(self) -> None: + self.assertEqual( + [(1, "文a", "中文abc 属于 HTML。")], + list(unspaced("中文abc 属于 HTML。\n")), + ) + + def test_a_table_row_is_not_an_html_tag(self) -> None: + line = "| 33-64 | 其他矿工 | <1% | - | <5 | 各类GPU | >400ms |\n" + self.assertEqual([(1, "类G", line.strip())], list(unspaced(line))) + + def test_display_math_blocks_are_skipped_whole(self) -> None: + text = "正文一行。\n\n$$\n\\text{原始Adam的权重衰减} = \\eta \\lambda \\theta\n$$\n\n正文二行。\n" + self.assertEqual([], list(unspaced(text))) + + +if __name__ == "__main__": + unittest.main()