From f315ef93b34b95c67557f0c73e999e806736f6cd Mon Sep 17 00:00:00 2001 From: yeasy Date: Mon, 24 Aug 2026 16:06:29 -0700 Subject: [PATCH] =?UTF-8?q?test:=20=E6=96=B0=E5=A2=9E=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E6=8E=92=E7=89=88=E9=97=A8=E7=A6=81=EF=BC=88=E4=B8=AD=E8=8B=B1?= =?UTF-8?q?=E6=96=87=E7=A9=BA=E6=A0=BC=20+=20=E5=90=AB=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E6=8B=AC=E5=8F=B7=E5=85=A8=E8=A7=92=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 今天这批 style 提交把两项排版统一到位，但**没有任何东西拦住下一段正文漂回去**：`check_project_rules.py` 看围栏与链接，`check_emphasis.py` 看加粗标记，工作区的 `format_checker.py` 看引号、围栏语言与标题空行——**行内排版此前一道门禁都没有**。漂移到什么程度是有实测的：同集群一本书的中英文边界曾有 45% 未空格、另一本的含中文括号是 50/50，而且都是在**同一个文件内**混用。 两条规则： - **规则一**：汉字不得与 `[A-Za-z0-9]` 直接相邻。`Harness的定义`、`第4章` 违规。 - **规则二**：括号内容含任何中文时必须用全角。`(用户命名空间映射)` 违规；**纯拉丁内容的半角括号一律合法**——`新建一个层 (Layer)`、`项目(Agent)的经验` 都不报，因为它们的内容里没有中文。 **刻意不检查**中文标点与拉丁字符之间的空格（`（如 UI、监控服务）`）——全集群一致不加，全角标点本身自带视觉留白。 **负向验证**（AGENTS.md 核心约定 4：不可能失败的检查不构成证据）：把本书扫改前（`ef262553~1`）的内容原样喂给这道门禁，报出 规则一 **28 处**、规则二 **338 处**；喂当前内容，报 **0 处**。 文件自带 9 个用例，其中 5 个专门钉住「不该报的别报」和「该报的必须报」：纯拉丁括号不报、行内代码/链接目标/裸 URL/HTML 属性/行间公式不报，而 `中文abc` 的**标签之间的文字**要报、`定价 $25/$125 per M tokens` 这种成对的 `$` **不得**被当成公式把中间的正文遮蔽掉、`| <1% | … | >400ms |` 这种表格行**不得**被当成一个 HTML 标签。后两条是扫改时真踩过的坑，掩码写松了会静默漏报。 占位符取私用区字符（既非汉字也非拉丁字符也非数字），且每条掩码正则都排除该区间——否则贪婪的 `\S+` 会把前一条规则刚放下的占位符一起吞掉。 --- tests/test_cjk_typography.py | 204 +++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 tests/test_cjk_typography.py diff --git a/tests/test_cjk_typography.py b/tests/test_cjk_typography.py new file mode 100644 index 0000000..5a2bcc2 --- /dev/null +++ b/tests/test_cjk_typography.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Guard the book's two Chinese-typography conventions. + +Neither rule is enforced anywhere else: ``check_project_rules.py`` looks at +fences and links, ``check_emphasis.py`` at bold markers, and the workspace +``format_checker.py`` at quotes, fence languages and heading spacing. Intra-line +typography had no gate at all, and it drifted exactly as far as you would +expect -- one sibling book reached 45% unspaced CJK/Latin boundaries and another +reached a 50/50 split on parentheses, both mixed *within single files*. A +one-off sweep normalised all fourteen books; this file is what stops the next +paragraph from drifting back. + +RULE 1 -- a CJK ideograph may not sit directly against ``[A-Za-z0-9]``. + ``Harness的定义`` / ``第4章`` are violations; ``Harness 的定义`` is not. + +RULE 2 -- a parenthesis whose content contains any CJK must be full-width. + ``(用户命名空间映射)`` is a violation; ``（用户命名空间映射）`` is not. + Latin-only parentheses are NOT touched: ``新建一个层 (Layer)`` and the + ``项目(Agent)的经验`` hugging style both stay legal, because their + content has no CJK. + +Deliberately NOT checked, because the whole cluster agrees on the opposite: +Chinese punctuation against Latin (``（如 UI、监控服务）``) needs no space -- +full-width punctuation already carries its own visual padding. + +Everything a reader never sees as prose is excluded before the scan: fenced code +blocks and display-math blocks are skipped whole, and inline code, math, link +and image targets, HTML and bare URLs are replaced with Private Use Area +placeholders -- characters that can never themselves be mistaken for a letter, a +digit or an ideograph. Every mask pattern excludes that range in turn, because a +greedy ``\\S+`` will otherwise swallow the placeholder its predecessor just left +behind. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKIP_DIRS = {".git", "_book", "node_modules", ".obsidian", "output", ".agent", "_site"} + +PUA_FIRST = 0xE000 +PUA = "\\ue000-\\uf8ff" +CJK = r"一-鿿㐀-䶿" +LATIN = r"A-Za-z0-9" + +FENCE = re.compile(r"^\s*(?:```|~~~)") +MATH_BLOCK = re.compile(r"^\s*\$\$\s*$") +LINK_DEF = re.compile(r"^\s*\[[^\]]+\]:\s*\S+") + +MASKS = ( + re.compile(r""), + re.compile(r"``[^`]+``"), + re.compile(r"`[^`]*`"), + re.compile(r"\$\$[^$]*\$\$"), + # only treat $...$ as math when it actually contains math syntax, so that a + # pair of prices ("$25/$125 per M tokens") is not read as one formula + re.compile(r"\$(?=[^$\n]*[\\^_{])[^$\n]*\$"), + re.compile(r"!?\]\([^)\s]*(?:\s+\"[^\"]*\")?\)"), + # a real tag, not "| <1% | - | <5 | 各类GPU | >400ms |" + re.compile(r"]*)?/?>"), + re.compile("(? str: + counter = [0] + + def repl(_match: "re.Match[str]") -> str: + counter[0] += 1 + return chr(PUA_FIRST + counter[0] - 1) + + for pattern in MASKS: + line = pattern.sub(repl, line) + return line + + +def prose_lines(text: str): + lines = text.split("\n") + in_front = bool(lines) and lines[0].strip() == "---" + in_fence = False + in_math = False + for number, line in enumerate(lines, start=1): + if in_front: + if number > 1 and line.strip() == "---": + in_front = False + continue + if FENCE.match(line): + in_fence = not in_fence + continue + if in_fence: + continue + if MATH_BLOCK.match(line): + in_math = not in_math + continue + if in_math or LINK_DEF.match(line): + continue + yield number, line + + +def markdown_files(root: Path = ROOT): + for path in sorted(root.rglob("*.md")): + if SKIP_DIRS.isdisjoint(path.relative_to(root).parts): + yield path + + +def unspaced(text: str): + """RULE 1 violations.""" + for number, line in prose_lines(text): + for match in ADJACENT.finditer(mask(line)): + yield number, match.group(0), line.strip()[:100] + + +def half_width_parens(text: str): + """RULE 2 violations.""" + for number, line in prose_lines(text): + for match in HALF_PAREN.finditer(mask(line)): + if re.search(f"[{CJK}]", match.group(0)): + yield number, match.group(0)[:40], line.strip()[:100] + + +class CjkTypographyTests(unittest.TestCase): + def test_cjk_and_latin_stay_apart(self) -> None: + found = [ + f"{path.relative_to(ROOT)}:{number}: {hit!r} in {snippet}" + for path in markdown_files() + for number, hit, snippet in unspaced(path.read_text(encoding="utf-8")) + ] + self.assertEqual( + [], found, "缺少中文与拉丁字符之间的半角空格：\n" + "\n".join(found[:40]) + ) + + def test_parens_holding_chinese_are_full_width(self) -> None: + found = [ + f"{path.relative_to(ROOT)}:{number}: {hit!r} in {snippet}" + for path in markdown_files() + for number, hit, snippet in half_width_parens(path.read_text(encoding="utf-8")) + ] + self.assertEqual( + [], found, "含中文的括号应使用全角：\n" + "\n".join(found[:40]) + ) + + def test_both_rules_can_actually_fail(self) -> None: + """A check that cannot fail is not evidence.""" + self.assertEqual([], list(unspaced("使用 Harness 的记忆子系统。\n"))) + self.assertEqual( + [(1, "s的", "使用 Harness的记忆子系统。")], + list(unspaced("使用 Harness的记忆子系统。\n")), + ) + self.assertEqual([(1, "第4", "第4 章")], list(unspaced("第4 章\n"))) + self.assertEqual([], list(half_width_parens("启用（用户命名空间映射）。\n"))) + self.assertEqual( + [(1, "(用户命名空间映射)", "启用 (用户命名空间映射)。")], + list(half_width_parens("启用 (用户命名空间映射)。\n")), + ) + + def test_latin_only_parens_are_left_alone(self) -> None: + for sample in ( + "在基础镜像之上新建一个层 (Layer)。", + "多个生产级智能体项目(Agent)的经验表明。", + "定义一组相关联的应用容器为一个项目 (project)。", + ): + self.assertEqual([], list(half_width_parens(sample + "\n")), sample) + + def test_non_prose_regions_are_excluded(self) -> None: + for sample in ( + "见 `mini_harness/核心.py` 说明。", + "参考 [第 4 章的运行时](04_runtime/README.md)。", + "地址是 https://example.com/中文path 这一条。", + '第4章示意图 的属性不算正文。', + "见 [文档](a.md) https://e.com/x 结束。", + "公式 $$L = (预测 - 真实)^2$$ 属于数学。", + "记 $a^*$ 与 $O_t$ 为最优解。", + ): + self.assertEqual([], list(unspaced(sample + "\n")), sample) + self.assertEqual([], list(half_width_parens(sample + "\n")), sample) + + def test_prices_are_not_mistaken_for_math(self) -> None: + """A pair of '$' around prose must not mask the prose between them.""" + line = "定价 $25/$125 per M tokens，另有 $1 亿额度给 Glasswing计划。\n" + self.assertEqual([(1, "g计", line.strip())], list(unspaced(line))) + + def test_html_text_content_is_still_prose(self) -> None: + self.assertEqual( + [(1, "文a", "中文abc 属于 HTML。")], + list(unspaced("中文abc 属于 HTML。\n")), + ) + + def test_a_table_row_is_not_an_html_tag(self) -> None: + line = "| 33-64 | 其他矿工 | <1% | - | <5 | 各类GPU | >400ms |\n" + self.assertEqual([(1, "类G", line.strip())], list(unspaced(line))) + + def test_display_math_blocks_are_skipped_whole(self) -> None: + text = "正文一行。\n\n$$\n\\text{原始Adam的权重衰减} = \\eta \\lambda \\theta\n$$\n\n正文二行。\n" + self.assertEqual([], list(unspaced(text))) + + +if __name__ == "__main__": + unittest.main()