v0.2 initial

README_HF.md

@@ -0,0 +1,57 @@
+---
+title: CHIP — Chinese High-density Instruction Protocol
+emoji: 🀄
+colorFrom: blue
+colorTo: orange
+sdk: gradio
+sdk_version: 4.44.0
+app_file: app.py
+pinned: true
+license: apache-2.0
+short_description: 数据驱动的中文 prompt 协议化压缩工具
+tags:
+  - chinese
+  - prompt-engineering
+  - llm
+  - tokenizer
+  - compression
+---
+
+# CHIP · 中文高密度提示协议
+
+把啰嗦的中文 prompt 自动压成结构化高密度形式 — **数据驱动,不是品味**。
+
+## 🎯 核心发现
+
+基于 9 个主流 tokenizer × 200 句 FLORES-200 平行语料的 1800 行实测:
+
+- **6 个国产 tokenizer 上中文 prompt token 数 ≤ 等价英文**
+  (Baichuan2: 中文省 12.5%,DeepSeek-V3: 省 8.4%,GLM-4: 省 7.6%)
+- **OpenAI cl100k 上中文比英文贵 73%**
+- **`###` 标签在所有 9 个 tokenizer 上都是 1 token**,完爆方括号方案
+
+## 🔧 怎么用
+
+在左侧粘贴你的中文 prompt,选择目标模型,点压缩。右侧会展示:
+
+1. **压缩后的 prompt**(可一键复制)
+2. **Token 统计**(在你选的 tokenizer 上节省了多少)
+3. **命中的规则**(audit trail,可追溯每条改动)
+
+## 📦 GitHub / pip
+
+```bash
+pip install chip-prompt
+```
+
+```python
+from chip import compress
+compress("请你帮我对下面这段文字进行一个全面的分析")
+# → '分析下面这段文字'
+```
+
+🔗 [GitHub repo](https://github.com/marcuscw/CHIP) · [SPEC.md](https://github.com/marcuscw/CHIP/blob/main/SPEC.md) · [Datasets](https://github.com/marcuscw/CHIP/tree/main/results)
+
+## ⚖️ License
+
+Apache-2.0

app.py

@@ -0,0 +1,665 @@
+"""
+app.py — CHIP HuggingFace Space 入口(美观版)
+
+部署: https://huggingface.co/spaces/<user>/CHIP
+"""
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+import gradio as gr
+import tiktoken
+
+from chip import Compressor
+
+# ============================================================
+# Tokenizer 计数器
+# ============================================================
+TOKENIZERS = {}
+try:
+    TOKENIZERS["GPT-4 (cl100k)"] = tiktoken.get_encoding("cl100k_base")
+    TOKENIZERS["GPT-4o (o200k)"] = tiktoken.get_encoding("o200k_base")
+except Exception as e:
+    print(f"[warn] tiktoken load failed: {e}")
+
+
+def _lazy_load_qwen():
+    from transformers import AutoTokenizer
+    return AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B", trust_remote_code=True)
+
+
+def _lazy_load_deepseek():
+    from transformers import AutoTokenizer
+    return AutoTokenizer.from_pretrained("deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True)
+
+
+_LAZY = {
+    "Qwen2.5": _lazy_load_qwen,
+    "DeepSeek-V2/V3": _lazy_load_deepseek,
+}
+
+
+def count_tokens(text: str, tokenizer_name: str) -> int:
+    if tokenizer_name in TOKENIZERS:
+        enc = TOKENIZERS[tokenizer_name]
+    elif tokenizer_name in _LAZY:
+        try:
+            tok = _LAZY[tokenizer_name]()
+            TOKENIZERS[tokenizer_name] = tok
+            enc = tok
+        except Exception:
+            return -1
+    else:
+        return -1
+
+    if hasattr(enc, "encode") and not hasattr(enc, "tokenize"):
+        return len(enc.encode(text))
+    return len(enc.encode(text, add_special_tokens=False))
+
+
+# ============================================================
+# 主压缩函数
+# ============================================================
+def run(text: str, target: str, use_l1: bool, use_l2: bool, use_l3: bool,
+        use_l4: bool, tokenizer_name: str, use_jieba: bool):
+    if not text.strip():
+        empty_card = "<div class='stat-card empty'>👈 请在左侧输入 prompt</div>"
+        return "", empty_card, "", ""
+
+    layers = []
+    for flag, name in [(use_l1, "L1"), (use_l2, "L2"), (use_l3, "L3"), (use_l4, "L4")]:
+        if flag:
+            layers.append(name)
+    if not layers:
+        layers = ["L1"]
+
+    if use_jieba:
+        os.environ["CHIP_USE_JIEBA"] = "1"
+    else:
+        os.environ.pop("CHIP_USE_JIEBA", None)
+    import chip.compressor as cc
+    cc._default_compressor = None
+
+    compressor = Compressor(target=target, layers=layers)
+    result = compressor.compress(text)
+
+    n_orig = count_tokens(text, tokenizer_name)
+    n_comp = count_tokens(result.compressed, tokenizer_name)
+
+    # ---- 漂亮的统计卡片 ----
+    if n_orig > 0 and n_comp > 0:
+        saving_pct = (1 - n_comp / n_orig) * 100
+        n_diff = n_orig - n_comp
+        if saving_pct > 0:
+            badge_class = "saving-badge good"
+            arrow = "↓"
+            verb = "节省"
+        elif saving_pct < 0:
+            badge_class = "saving-badge bad"
+            arrow = "↑"
+            verb = "增加"
+        else:
+            badge_class = "saving-badge neutral"
+            arrow = "→"
+            verb = "持平"
+
+        char_orig, char_comp = len(text), len(result.compressed)
+        char_pct = (1 - char_comp / max(char_orig, 1)) * 100
+
+        stats_html = f"""
+<div class="stats-grid">
+  <div class="stat-card primary">
+    <div class="stat-label">Token 节省</div>
+    <div class="stat-value {badge_class}">
+      <span class="arrow">{arrow}</span> {abs(saving_pct):.1f}%
+    </div>
+    <div class="stat-sub">{verb} {abs(n_diff)} token · 在 {tokenizer_name}</div>
+  </div>
+  <div class="stat-card">
+    <div class="stat-label">原文</div>
+    <div class="stat-value muted">{n_orig}</div>
+    <div class="stat-sub">token · {char_orig} 字符</div>
+  </div>
+  <div class="stat-card">
+    <div class="stat-label">压缩后</div>
+    <div class="stat-value emphasis">{n_comp}</div>
+    <div class="stat-sub">token · {char_comp} 字符</div>
+  </div>
+  <div class="stat-card">
+    <div class="stat-label">字符压缩</div>
+    <div class="stat-value muted">{char_pct:+.0f}%</div>
+    <div class="stat-sub">字符数变化(供参考,token 才重要)</div>
+  </div>
+</div>"""
+    else:
+        stats_html = (
+            "<div class='stat-card empty'>"
+            "⚠️ Token 计数不可用(可能是 tokenizer 加载失败)"
+            "</div>"
+        )
+
+    # ---- 漂亮的规则面板 ----
+    if result.applied_rules:
+        rules_items = []
+        for r in result.applied_rules:
+            # 解析 rule id 和命中次数
+            parts = r.split("×")
+            rid = parts[0]
+            count = parts[1] if len(parts) > 1 else "1"
+            # 按层着色
+            layer_color = {"L1": "#3B7DD8", "L2": "#E6883C", "L3": "#7C3AED", "L4": "#10B981"}
+            color = "#888"
+            for l, c in layer_color.items():
+                if l in rid:
+                    color = c
+                    break
+            badge = (
+                f"<span class='rule-pill' style='--rule-color:{color}'>"
+                f"<span class='rule-id'>{rid}</span>"
+                f"<span class='rule-count'>×{count}</span>"
+                f"</span>"
+            )
+            rules_items.append(badge)
+        rules_html = (
+            "<div class='rules-header'>🎯 命中规则 "
+            f"<span class='rules-count'>{len(result.applied_rules)} 条</span></div>"
+            "<div class='rules-pills'>" + " ".join(rules_items) + "</div>"
+        )
+    else:
+        rules_html = (
+            "<div class='rules-header empty-rules'>"
+            "📭 没有规则匹配 — 这段 prompt 已经够紧凑了"
+            "</div>"
+        )
+
+    return result.compressed, stats_html, rules_html, result.compressed
+
+
+# ============================================================
+# 示例
+# ============================================================
+EXAMPLES = [
+    [
+        "请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些改进的建议",
+        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
+    ],
+    [
+        "请你扮演一位资深 Python 工程师,对下面的代码进行 code review,并以 JSON 格式输出结果",
+        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", True,
+    ],
+    [
+        "因为最近在下雨,所以路面变得很滑,因此开车的时候需要特别注意安全",
+        "qwen2.5", True, True, False, True, "GPT-4 (cl100k)", False,
+    ],
+    [
+        "Role: 资深产品经理\n任务: 评估这个需求,大家都知道产品决策需要数据支持",
+        "deepseek_v3", True, True, True, True, "GPT-4 (cl100k)", False,
+    ],
+]
+
+
+# ============================================================
+# 自定义 CSS — 让 demo 看起来不像默认的 gradio
+# ============================================================
+CUSTOM_CSS = """
+/* ===== 全局 ===== */
+.gradio-container {
+  max-width: 1280px !important;
+  margin: 0 auto !important;
+}
+
+/* ===== Hero 区 ===== */
+.hero {
+  text-align: center;
+  padding: 36px 16px 12px;
+  background: linear-gradient(135deg, rgba(59,125,216,0.08) 0%, rgba(230,136,60,0.08) 100%);
+  border-radius: 16px;
+  margin-bottom: 24px;
+  border: 1px solid rgba(0,0,0,0.05);
+}
+.hero h1 {
+  font-size: 2.4em !important;
+  margin: 0 !important;
+  background: linear-gradient(135deg, #3B7DD8, #E6883C);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+  background-clip: text;
+  font-weight: 700 !important;
+  letter-spacing: -0.02em;
+}
+.hero .tagline {
+  font-size: 1.1em;
+  color: #555;
+  margin-top: 8px;
+}
+.hero .subtitle {
+  font-size: 0.95em;
+  color: #777;
+  max-width: 760px;
+  margin: 12px auto 0;
+  line-height: 1.6;
+}
+.hero .badges {
+  margin-top: 16px;
+  display: flex;
+  gap: 8px;
+  justify-content: center;
+  flex-wrap: wrap;
+}
+.hero .badge {
+  display: inline-flex;
+  align-items: center;
+  padding: 4px 10px;
+  background: rgba(255,255,255,0.7);
+  border: 1px solid rgba(0,0,0,0.08);
+  border-radius: 6px;
+  font-size: 0.85em;
+  color: #444;
+  text-decoration: none;
+  transition: all 0.2s;
+}
+.hero .badge:hover {
+  transform: translateY(-1px);
+  box-shadow: 0 2px 8px rgba(0,0,0,0.08);
+}
+
+/* ===== Key facts ===== */
+.key-facts {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+  gap: 12px;
+  margin: 24px 0;
+}
+.fact {
+  padding: 14px 18px;
+  background: white;
+  border-radius: 10px;
+  border-left: 3px solid var(--fact-color, #3B7DD8);
+  box-shadow: 0 1px 3px rgba(0,0,0,0.05);
+}
+.fact-label {
+  font-size: 0.75em;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #888;
+  font-weight: 600;
+}
+.fact-value {
+  font-size: 1.5em;
+  font-weight: 700;
+  color: var(--fact-color, #3B7DD8);
+  margin-top: 4px;
+}
+.fact-detail {
+  font-size: 0.85em;
+  color: #666;
+  margin-top: 2px;
+}
+
+/* ===== 统计卡片 ===== */
+.stats-grid {
+  display: grid;
+  grid-template-columns: 2fr 1fr 1fr 1fr;
+  gap: 12px;
+  margin: 8px 0;
+}
+.stat-card {
+  background: white;
+  border-radius: 10px;
+  padding: 14px 16px;
+  border: 1px solid rgba(0,0,0,0.06);
+  transition: all 0.2s;
+}
+.stat-card:hover {
+  box-shadow: 0 2px 12px rgba(0,0,0,0.08);
+  transform: translateY(-1px);
+}
+.stat-card.primary {
+  background: linear-gradient(135deg, #FAFBFF, #F5F7FB);
+  border: 1px solid #DDE5F0;
+}
+.stat-card.empty {
+  text-align: center;
+  padding: 28px;
+  background: #F8F9FB;
+  color: #888;
+  border: 1px dashed #DDD;
+  font-size: 0.95em;
+}
+.stat-label {
+  font-size: 0.75em;
+  text-transform: uppercase;
+  letter-spacing: 0.06em;
+  color: #888;
+  font-weight: 600;
+}
+.stat-value {
+  font-size: 1.6em;
+  font-weight: 700;
+  margin-top: 4px;
+  display: flex;
+  align-items: baseline;
+  gap: 6px;
+}
+.stat-value.muted { color: #999; font-size: 1.4em; }
+.stat-value.emphasis { color: #10B981; font-size: 1.8em; }
+.saving-badge.good { color: #10B981; }
+.saving-badge.bad { color: #E6883C; }
+.saving-badge.neutral { color: #888; }
+.arrow { font-weight: 700; }
+.stat-sub {
+  font-size: 0.78em;
+  color: #777;
+  margin-top: 2px;
+}
+
+/* ===== 规则面板 ===== */
+.rules-header {
+  font-size: 0.95em;
+  font-weight: 600;
+  color: #444;
+  margin: 16px 0 8px;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.rules-count {
+  font-size: 0.78em;
+  background: #EFF2F7;
+  padding: 2px 10px;
+  border-radius: 10px;
+  color: #555;
+  font-weight: 500;
+}
+.rules-pills {
+  display: flex;
+  gap: 6px;
+  flex-wrap: wrap;
+  padding: 4px 0;
+}
+.rule-pill {
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  padding: 4px 10px;
+  background: white;
+  border: 1px solid var(--rule-color);
+  color: var(--rule-color);
+  border-radius: 999px;
+  font-size: 0.82em;
+  font-family: 'JetBrains Mono', 'Fira Code', monospace;
+  font-weight: 500;
+  transition: all 0.2s;
+}
+.rule-pill:hover {
+  background: var(--rule-color);
+  color: white;
+}
+.rule-count {
+  font-size: 0.85em;
+  opacity: 0.8;
+}
+.empty-rules {
+  font-style: italic;
+  color: #888;
+  background: #F8F9FB;
+  padding: 14px;
+  border-radius: 8px;
+  border: 1px dashed #DDD;
+  text-align: center;
+}
+
+/* ===== 主操作按钮 ===== */
+button.primary {
+  background: linear-gradient(135deg, #3B7DD8 0%, #5B9BE5 100%) !important;
+  border: none !important;
+  font-size: 1.05em !important;
+  letter-spacing: 0.02em !important;
+  transition: all 0.2s !important;
+}
+button.primary:hover {
+  transform: translateY(-1px);
+  box-shadow: 0 4px 12px rgba(59,125,216,0.3) !important;
+}
+
+/* ===== Footer ===== */
+.footer {
+  margin-top: 36px;
+  padding: 20px 0;
+  border-top: 1px solid rgba(0,0,0,0.06);
+  color: #777;
+  font-size: 0.88em;
+  text-align: center;
+}
+.footer a { color: #3B7DD8; text-decoration: none; }
+.footer a:hover { text-decoration: underline; }
+
+/* ===== 暗色模式适配 ===== */
+.dark .hero {
+  background: linear-gradient(135deg, rgba(59,125,216,0.15), rgba(230,136,60,0.15));
+  border-color: rgba(255,255,255,0.05);
+}
+.dark .hero .tagline { color: #BBB; }
+.dark .hero .subtitle { color: #999; }
+.dark .hero .badge {
+  background: rgba(255,255,255,0.05);
+  border-color: rgba(255,255,255,0.1);
+  color: #CCC;
+}
+.dark .stat-card { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.08); }
+.dark .stat-card.primary { background: rgba(59,125,216,0.08); }
+.dark .stat-card.empty { background: rgba(255,255,255,0.03); color: #888; border-color: rgba(255,255,255,0.1); }
+.dark .stat-sub { color: #888; }
+.dark .fact { background: rgba(255,255,255,0.03); }
+.dark .fact-detail { color: #888; }
+.dark .rules-count { background: rgba(255,255,255,0.06); color: #BBB; }
+.dark .rule-pill { background: rgba(255,255,255,0.03); }
+.dark .empty-rules { background: rgba(255,255,255,0.03); border-color: rgba(255,255,255,0.1); }
+
+/* ===== 响应式 ===== */
+@media (max-width: 768px) {
+  .stats-grid { grid-template-columns: 1fr 1fr; }
+  .key-facts { grid-template-columns: 1fr; }
+  .hero h1 { font-size: 1.8em !important; }
+}
+"""
+
+
+# ============================================================
+# UI
+# ============================================================
+HERO_HTML = """
+<div class="hero">
+  <h1>🀄 CHIP</h1>
+  <div class="tagline">Chinese High-density Instruction Protocol · 中文高密度提示协议</div>
+  <div class="subtitle">
+    把啰嗦的中文 prompt 自动压成结构化高密度形式 — <strong>数据驱动,不是品味</strong>
+  </div>
+  <div class="badges">
+    <a class="badge" href="https://github.com/luancy1208/CHIP" target="_blank">⭐ GitHub</a>
+    <a class="badge" href="https://github.com/luancy1208/CHIP/blob/main/SPEC.md" target="_blank">📄 SPEC</a>
+    <a class="badge" href="https://github.com/luancy1208/CHIP/tree/main/results" target="_blank">📊 实测数据</a>
+    <span class="badge">Apache-2.0</span>
+    <span class="badge">v0.2 · 23 tests passing</span>
+  </div>
+</div>
+
+<div class="key-facts">
+  <div class="fact" style="--fact-color:#3B7DD8">
+    <div class="fact-label">实测数据</div>
+    <div class="fact-value">1800+ 行</div>
+    <div class="fact-detail">9 tokenizer × 200 句 FLORES-200</div>
+  </div>
+  <div class="fact" style="--fact-color:#10B981">
+    <div class="fact-label">国产模型上中文</div>
+    <div class="fact-value">省 12.5%</div>
+    <div class="fact-detail">Baichuan2 token / 等价英文</div>
+  </div>
+  <div class="fact" style="--fact-color:#E6883C">
+    <div class="fact-label">cl100k 上中文</div>
+    <div class="fact-value">贵 73%</div>
+    <div class="fact-detail">所以 CHIP 的主战场是国产模型</div>
+  </div>
+  <div class="fact" style="--fact-color:#7C3AED">
+    <div class="fact-label">### 标签</div>
+    <div class="fact-value">1 token</div>
+    <div class="fact-detail">在所有 9 个 tokenizer 上 — 完爆方括号</div>
+  </div>
+</div>
+"""
+
+
+FOOTER_HTML = """
+<div class="footer">
+  <p>
+    🀄 CHIP v0.2 · Built with care for Chinese prompt engineering ·
+    <a href="https://github.com/luancy1208/CHIP" target="_blank">GitHub</a> ·
+    <a href="https://github.com/luancy1208/CHIP/issues" target="_blank">反馈 Issue</a>
+  </p>
+  <p style="margin-top:8px; font-size:0.82em; opacity:0.7">
+    数据来源: FLORES-200 dev (n=200) · 200 个 HSK 5/6 高频成语 · 45 个常见标记符号 · 实测于 9 个 tokenizer
+  </p>
+</div>
+"""
+
+
+with gr.Blocks(
+    title="CHIP — 中文高密度提示协议",
+    theme=gr.themes.Soft(
+        primary_hue="blue",
+        secondary_hue="orange",
+        neutral_hue="slate",
+        font=["system-ui", "-apple-system", "Segoe UI", "Helvetica Neue", "sans-serif"],
+    ),
+    css=CUSTOM_CSS,
+) as demo:
+    gr.HTML(HERO_HTML)
+
+    with gr.Row(equal_height=True):
+        # ------- 输入侧 -------
+        with gr.Column(scale=1):
+            gr.Markdown("### 📝 输入 prompt")
+            inp = gr.Textbox(
+                show_label=False,
+                lines=10,
+                placeholder="把啰嗦的中文 prompt 粘贴进来,看看 CHIP 能压到多紧...\n\n💡 试试:'请你帮我对下面这段文字进行一个全面的分析,如果可以的话麻烦你给出一些建议'",
+                container=False,
+            )
+
+            with gr.Accordion("⚙️ 高级设置", open=False):
+                with gr.Row():
+                    target = gr.Dropdown(
+                        ["qwen2.5", "deepseek_v3", "glm4", "cl100k", "o200k"],
+                        value="qwen2.5",
+                        label="🎯 目标模型",
+                        info="影响 L3 成语压缩等 target-aware 决策",
+                    )
+                    tokenizer_name = gr.Dropdown(
+                        list(TOKENIZERS.keys()) + list(_LAZY.keys()),
+                        value="GPT-4 (cl100k)",
+                        label="🔢 计数 tokenizer",
+                        info="决定 token 数怎么算(国产 tokenizer 首次需下载)",
+                    )
+
+                gr.Markdown("**压缩层** — 默认 L1+L2+L4(保险);L3 仅在国产模型上有意义")
+                with gr.Row():
+                    use_l1 = gr.Checkbox(value=True, label="L1 词法",
+                                          info="套话剪枝 (~1.3-1.5×)")
+                    use_l2 = gr.Checkbox(value=True, label="L2 句法",
+                                          info="模式重排 (~2-3×)")
+                    use_l3 = gr.Checkbox(value=False, label="L3 成语",
+                                          info="长描述→成语")
+                    use_l4 = gr.Checkbox(value=True, label="L4 协议",
+                                          info="### 标签归一化")
+
+                use_jieba = gr.Checkbox(
+                    value=False,
+                    label="🚀 jieba NP 增强模式",
+                    info="复杂角色提取场景效果更好(默认关闭)",
+                )
+
+            btn = gr.Button("🔥  压缩", variant="primary", size="lg",
+                            elem_classes="primary")
+
+        # ------- 输出侧 -------
+        with gr.Column(scale=1):
+            gr.Markdown("### ✨ 压缩结果")
+            try:
+                out = gr.Textbox(
+                    show_label=False,
+                    lines=10,
+                    interactive=False,
+                    show_copy_button=True,
+                    container=False,
+                )
+            except TypeError:
+                out = gr.Textbox(show_label=False, lines=10,
+                                  interactive=False, container=False)
+            stats_panel = gr.HTML(
+                "<div class='stat-card empty'>👈 在左侧输入并点击压缩按钮</div>"
+            )
+            rules_panel = gr.HTML()
+
+    # 隐藏的 textbox 用于触发示例(因为示例需要更新所有 input)
+    hidden_dup = gr.Textbox(visible=False)
+
+    gr.Examples(
+        examples=EXAMPLES,
+        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
+        label="📌 试试这些示例(点击自动填充并运行)",
+        examples_per_page=4,
+    )
+
+    with gr.Accordion("📖 关于 CHIP / 协议设计要点", open=False):
+        gr.Markdown("""
+**核心发现(基于 1800 行实测数据)**
+
+| Tokenizer | ZH/EN ratio | 中文相对英文 |
+|---|---|---|
+| **baichuan2** 🟦 | 0.875 | 中文省 12.5% |
+| **deepseek_v3** 🟦 | 0.916 | 中文省 8.4% |
+| **glm4** 🟦 | 0.924 | 中文省 7.6% |
+| qwen2.5/3 🟦 | 0.988 | 持平 |
+| **o200k** (GPT-4o) 🟧 | 1.163 | 中文贵 16.3% |
+| **cl100k** (GPT-4) 🟧 | 1.731 | 中文贵 73.1% |
+
+🟦 = 国产 tokenizer  ·  🟧 = OpenAI tokenizer
+
+---
+
+**4 层压缩架构**
+
+- **L1 词法层** · 啰嗦套话 → 紧凑动宾,纯正则,~1.3-1.5×
+- **L2 句法层** · 模式重排 + 列表化,~2-3×
+- **L3 成语层** · 长描��� → 成语(实测白名单),仅国产模型默认关闭
+- **L4 协议层** · 标签归一化为 `### 标题`(实测全 tokenizer 1 token)
+
+**不主张的事**(诚实声明)
+
+- ❌ 不主张"中文因为信息密度高所以更省 token" — Mythbuster 2026 在 SWE-bench 上证伪
+- ❌ 不主张"中文 prompt 让模型更聪明" — 在英文中心模型上常常相反
+- ❌ 不主张"全程使用文言文压缩" — LLM 对生僻文言虚词理解不稳定
+
+CHIP 主张的是:**在符合中文训练分布的国产模型上,通过协议化压缩可在 token 经济性、可读性、可审计性三个维度同时提供工程价值。**
+""")
+
+    gr.HTML(FOOTER_HTML)
+
+    # 事件
+    btn.click(
+        run,
+        inputs=[inp, target, use_l1, use_l2, use_l3, use_l4, tokenizer_name, use_jieba],
+        outputs=[out, stats_panel, rules_panel, hidden_dup],
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name=os.getenv("HOST", "0.0.0.0"),
+        server_port=int(os.getenv("PORT", 7860)),
+        share=os.getenv("SHARE") == "1",
+    )

chip/__init__.py

@@ -0,0 +1,15 @@
+"""
+chip — Chinese High-density Instruction Protocol
+
+主要 API:
+    >>> from chip import compress
+    >>> compress("请你帮我总结一下这段文字的核心观点是什么")
+    '总结此文核心观点'
+
+    >>> compress(text, target="qwen2.5", layers=("L1", "L2"))
+"""
+
+from chip.compressor import compress, Compressor, CompressionResult
+
+__version__ = "0.1.0"
+__all__ = ["compress", "Compressor", "CompressionResult", "__version__"]

chip/cli.py

@@ -0,0 +1,54 @@
+"""
+chip.cli — 命令行入口
+
+安装后:
+    chip "请你帮我总结一下这段文字"
+    echo "..." | chip
+    chip --target qwen2.5 --layers L1 L2 --diff "..."
+"""
+from __future__ import annotations
+import argparse, sys
+
+from chip.compressor import Compressor
+
+
+def main():
+    ap = argparse.ArgumentParser(prog="chip",
+        description="CHIP — Chinese High-density Instruction Protocol compressor")
+    ap.add_argument("text", nargs="?",
+                    help="prompt text (or read from stdin if omitted)")
+    ap.add_argument("--target", default="qwen2.5",
+                    choices=["qwen2.5", "cl100k", "o200k", "deepseek_v3", "glm4"],
+                    help="target tokenizer (decides protocol track)")
+    ap.add_argument("--layers", nargs="+", default=["L1", "L2", "L4"],
+                    choices=["L1", "L2", "L3", "L4"],
+                    help="L1=词法 L2=句法 L3=成语(需国产模型) L4=协议归一化")
+    ap.add_argument("--diff", action="store_true",
+                    help="show original / compressed / rules side-by-side")
+    ap.add_argument("--rules", default=None,
+                    help="custom rules.yaml path")
+    args = ap.parse_args()
+
+    if args.text:
+        text = args.text
+    else:
+        text = sys.stdin.read()
+        if not text.strip():
+            ap.print_help()
+            sys.exit(1)
+
+    kwargs = {}
+    if args.rules:
+        kwargs["rules_path"] = args.rules
+    compressor = Compressor(target=args.target, layers=args.layers, **kwargs)
+    result = compressor.compress(text)
+
+    if args.diff:
+        print(result.diff())
+        print(f"\n字符压缩率: {result.char_ratio:.2%}  ({len(result.original)} → {len(result.compressed)})")
+    else:
+        print(result.compressed)
+
+
+if __name__ == "__main__":
+    main()

chip/compressor.py

@@ -0,0 +1,334 @@
+"""
+chip.compressor
+================
+CHIP 主压缩器。设计原则:
+    1. 协议是文本,不是模型 — 不依赖 LLM 调用,纯规则可跑
+    2. 双轨 — Qwen 轨用中文方括号,cl100k 轨用 XML/Markdown
+    3. 可逆 — 保留命名实体、数字、代码、URL 不动
+    4. 可审计 — 每条改动可追溯到 rules.yaml 的某条规则
+
+当前实现层级:
+    L1 (lex)   — 词法替换:啰嗦套话 → 紧凑动宾,纯正则,~1.3-1.5x 压缩
+    L2 (syn)   — 句法重排:虚词替换、列表化,需 jieba 分词,~2-3x
+    L3 (idiom) — 成语压缩(基于实测白名单),需 target 是国产 tokenizer
+    L4 (proto) — 协议层归一化,统一为 ### 标签
+
+NP-aware 角色提取(可选):
+    L2-022 默认用正则,在含空格的复合 NP 上偶有截断。
+    设环境变量 CHIP_USE_JIEBA=1 启用 jieba 增强版。
+"""
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterable
+
+import yaml
+
+
+# ============ 数据类 ============
+@dataclass
+class Rule:
+    """一条 CHIP 转换规则。"""
+    id: str
+    layer: str           # "L1" | "L2" | "L3" | "L4"
+    pattern: str         # 正则
+    replacement: str
+    description: str = ""
+    saves: int = 0       # 在参考 tokenizer 上预估省多少 token
+    risk: str = "low"    # low | mid | high
+    flags: int = 0
+    _compiled: re.Pattern = field(default=None, repr=False)
+
+    def compile(self):
+        if self._compiled is None:
+            self._compiled = re.compile(self.pattern, self.flags)
+        return self._compiled
+
+
+@dataclass
+class CompressionResult:
+    """压缩结果,带 audit trail。"""
+    original: str
+    compressed: str
+    applied_rules: list[str]   # 命中的 rule id 列表
+    target: str                # tokenizer 名
+    layers: tuple
+
+    @property
+    def char_ratio(self) -> float:
+        return len(self.compressed) / max(len(self.original), 1)
+
+    def diff(self) -> str:
+        """简单的并排展示。"""
+        return f"原:  {self.original}\n压:  {self.compressed}\n规则: {', '.join(self.applied_rules) or '(none)'}"
+
+
+# ============ 规则加载 ============
+DEFAULT_RULES_PATH = Path(__file__).parent / "rules" / "rules.yaml"
+
+
+def load_rules(path: Path | str = DEFAULT_RULES_PATH) -> list[Rule]:
+    """从 yaml 加载规则。"""
+    path = Path(path)
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+
+    rules = []
+    for item in data.get("rules", []):
+        flags = 0
+        for flag_name in item.get("flags", []):
+            flags |= getattr(re, flag_name.upper(), 0)
+        rules.append(Rule(
+            id=item["id"],
+            layer=item["layer"],
+            pattern=item["pattern"],
+            replacement=item.get("replacement", ""),
+            description=item.get("description", ""),
+            saves=item.get("saves", 0),
+            risk=item.get("risk", "low"),
+            flags=flags,
+        ))
+    return rules
+
+
+# ============ 保护性 mask ============
+# 这些 pattern 命中的子串会先被替换成占位符,跑完规则后再还原。
+# 防止规则误改专有名词、URL、代码、数字。
+PROTECT_PATTERNS = [
+    ("URL",   re.compile(r"https?://\S+")),
+    ("CODE",  re.compile(r"```[\s\S]*?```|`[^`\n]+`")),
+    ("NUM",   re.compile(r"\d+(?:\.\d+)?(?:%|km|kg|m|s|°C)?")),
+    ("EMAIL", re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+")),
+    # 双引号包裹的引文(用户原话)
+    ("QUOTE", re.compile(r"[\"\u201c][^\"\u201d]+[\"\u201d]")),
+]
+
+
+# 占位符前缀用一个不会出现在自然中文里、且不会被 PROTECT_PATTERNS 命中的 token
+_PH_OPEN = "\u2983"  # ⦃
+_PH_CLOSE = "\u2984"  # ⦄
+_PH_RE = re.compile(rf"{_PH_OPEN}\d+{_PH_CLOSE}")
+
+
+def _mask(text: str) -> tuple[str, list[tuple[str, str]]]:
+    """把不可压缩片段替换成 ⦃i⦄ 占位符,返回 (masked, mappings)。
+
+    关键:每次 sub 时跳过已经 mask 过的占位符,避免嵌套替换。
+    """
+    mappings = []
+    masked = text
+
+    def make_sub():
+        def _sub(m):
+            # 如果 match 整体落在已有占位符内,跳过
+            content = m.group(0)
+            if _PH_RE.fullmatch(content):
+                return content
+            i = len(mappings)
+            placeholder = f"{_PH_OPEN}{i}{_PH_CLOSE}"
+            mappings.append((placeholder, content))
+            return placeholder
+        return _sub
+
+    for tag, pat in PROTECT_PATTERNS:
+        masked = pat.sub(make_sub(), masked)
+    return masked, mappings
+
+
+def _unmask(text: str, mappings: list[tuple[str, str]]) -> str:
+    # 反向替换避免 ⦃1⦄ 误替换 ⦃10⦄
+    for placeholder, original in reversed(mappings):
+        text = text.replace(placeholder, original)
+    return text
+
+
+# ============ 主类 ============
+class Compressor:
+    """可重用的压缩器实例。"""
+
+    def __init__(self,
+                 rules_path: Path | str = DEFAULT_RULES_PATH,
+                 target: str = "qwen2.5",
+                 layers: Iterable[str] = ("L1", "L2", "L4")):
+        """
+        Args:
+            target: 目标 tokenizer,影响成语压缩等 target-aware 决策
+            layers: 启用的压缩层
+                - L1: 词法层(套话剪枝),保险,默认开
+                - L2: 句法层(模式重排),保险,默认开
+                - L3: 成语层(语义压缩),需 target 是国产 tokenizer 才有意义,默认关
+                - L4: 协议层归一化(### 标题统一),无害,默认开
+        """
+        self.rules = load_rules(rules_path)
+        self.target = target
+        self.layers = tuple(layers)
+        # 预编译
+        for r in self.rules:
+            r.compile()
+
+    def compress(self, text: str) -> CompressionResult:
+        original = text
+
+        # 可选:jieba 增强角色提取 (pre-process,优先于 L2-022 的纯正则)
+        applied_pre = []
+        if os.getenv("CHIP_USE_JIEBA") == "1" and "L2" in self.layers:
+            text, jieba_applied = _jieba_role_extract(text)
+            if jieba_applied:
+                applied_pre.append("L2-022J(jieba)")
+
+        masked, mappings = _mask(text)
+        applied = list(applied_pre)
+
+        for rule in self.rules:
+            if rule.layer not in self.layers:
+                continue
+            new_text, n = rule._compiled.subn(rule.replacement, masked)
+            if n > 0:
+                applied.append(f"{rule.id}×{n}")
+                masked = new_text
+
+        # 收尾:多余空白、连续标点
+        masked = re.sub(r"[ \t]+", " ", masked)
+        masked = re.sub(r"\s*\n\s*\n\s*\n+", "\n\n", masked)
+
+        # 协议层留下的孤立标点清理(L2-022 等会留下 "\n,xxx")
+        masked = re.sub(r"\n[,,;;。.\s]+", "\n", masked)
+        masked = re.sub(r"^[,,;;]+\s*", "", masked, flags=re.MULTILINE)
+
+        masked = masked.strip()
+        compressed = _unmask(masked, mappings)
+
+        return CompressionResult(
+            original=original,
+            compressed=compressed,
+            applied_rules=applied,
+            target=self.target,
+            layers=self.layers,
+        )
+
+
+# ============ 便捷函数 ============
+_default_compressor = None
+
+
+def compress(text: str,
+             target: str = "qwen2.5",
+             layers: Iterable[str] = ("L1", "L2", "L4"),
+             return_result: bool = False) -> str | CompressionResult:
+    """简便入口。
+
+    >>> compress("请帮我总结一下这段文字")
+    '总结一下这段文字'
+
+    >>> compress("...", layers=["L1","L2","L3","L4"])  # 启用所有层(包括成语)
+
+    >>> r = compress("...", return_result=True)
+    >>> print(r.diff())
+    """
+    global _default_compressor
+    key = (target, tuple(layers))
+    if _default_compressor is None or _default_compressor[0] != key:
+        _default_compressor = (key, Compressor(target=target, layers=layers))
+    result = _default_compressor[1].compress(text)
+    return result if return_result else result.compressed
+
+
+# ============ jieba NP 提取(可选增强) ============
+_jieba_loaded = False
+
+
+def _ensure_jieba():
+    """懒加载 jieba。"""
+    global _jieba_loaded
+    if _jieba_loaded:
+        return True
+    try:
+        import jieba.posseg as pseg  # noqa: F401
+        _jieba_loaded = True
+        return True
+    except ImportError:
+        return False
+
+
+# 角色扮演的触发短语 — jieba 用它定位
+_ROLE_PREFIX_RE = re.compile(
+    r"请\s*(?:你)?\s*扮演\s*(?:一(?:个|位))?\s*"
+)
+
+
+def _jieba_role_extract(text: str) -> tuple[str, bool]:
+    """用 jieba 词性标注提取最长名词短语作为角色描述。
+
+    替换 L2-022 的纯正则 lookahead 实现 — 后者在以下场景失败:
+        - 角色描述非常长且无标点结尾
+        - 角色描述被句中的连词意外截断("...然后..." 这种)
+
+    策略:
+        1. 找到 "请你扮演[一位]" 触发短语
+        2. 从触发短语后开始,jieba.posseg 切分
+        3. 贪婪收集 NP token,直到遇到 hard-stop:
+           - 连词 c (然后/接着/以及)
+           - 介词 p (对/把/为)
+           - 动词 v (但 vn 动名词允许)
+           - 句末标点 w (。;,等)
+        4. 助词 'uj/u/ul'(的/地/得)、空格、英文都允许进入 NP
+    """
+    if not _ensure_jieba():
+        return text, False
+
+    import jieba.posseg as pseg
+
+    m = _ROLE_PREFIX_RE.search(text)
+    if not m:
+        return text, False
+
+    head = text[:m.start()]
+    body = text[m.end():]
+    if not body:
+        return text, False
+
+    words = list(pseg.cut(body))
+
+    # NP 定义:最长前缀,直到遇到硬终止
+    # HARD_STOP:动词(非 vn)、连词、介词、标点
+    # ALLOW_IN_NP:名词、形容词、英文、数字、量词、助词(的/地/得)、空格
+    np_chars = []
+    cumlen = 0
+    rest_start = 0
+    found_np_core = False  # 是否已经收到名词或形容词(NP 核心)
+
+    for w, flag in words:
+        # hard stop 条件
+        is_hard_stop = (
+            flag == "w"                                 # 标点
+            or w in {",", ",", "。", ".", ";", ";", ":", ":", "、", "\n"}
+            or flag == "c"                              # 连词
+            or flag == "p"                              # 介词
+            or (flag.startswith("v") and flag != "vn")  # 真动词(非动名词)
+        )
+        if is_hard_stop and found_np_core:
+            rest_start = cumlen
+            break
+
+        # 在 NP 内
+        np_chars.append(w)
+        cumlen += len(w)
+        if flag.startswith("n") or flag.startswith("a") or flag == "eng":
+            found_np_core = True
+    else:
+        # 遍历完了,整个 body 都是 NP
+        rest_start = cumlen
+
+    np_str = "".join(np_chars).strip()
+    if not np_str or len(np_str) < 2 or not found_np_core:
+        return text, False
+
+    rest = body[rest_start:]
+    new_text = f"{head}\n### 角色\n{np_str}\n{rest}"
+    # 清理紧跟在角色块后的孤立标点
+    new_text = re.sub(r"\n[,,;;。.]+", "\n", new_text)
+    new_text = new_text.strip()
+    return new_text, True

chip/rules/rules.yaml

@@ -0,0 +1,418 @@
+# CHIP Compression Rules v0.2 (2025-05-01)
+# ==========================================
+# v0.2 vs v0.1 主要变化:
+#   - 标签层从 [角:X] / 【任】 改为 ### 角色 (实测全 tokenizer 1 token,完爆方括号)
+#   - 新增 L3 成语层(基于 idiom_whitelist.json 实测)
+#   - 新增 L4 协议层(归一化用户已有的标签)
+
+rules:
+  # ============================================================
+  # L1: 词法层 — 啰嗦套话剪枝
+  # ============================================================
+  - id: L1-001
+    layer: L1
+    pattern: "请你?帮我?"
+    replacement: ""
+    saves: 2
+    risk: low
+    description: "客套语 '请你帮我' / '请帮我' → 空"
+
+  - id: L1-002
+    layer: L1
+    pattern: "麻烦你?"
+    replacement: ""
+    saves: 2
+    risk: low
+
+  - id: L1-003
+    layer: L1
+    pattern: "如果可以的话[,,]?"
+    replacement: ""
+    saves: 3
+    risk: low
+
+  - id: L1-004
+    layer: L1
+    pattern: "(?:能不能|可不可以|可以|能)(?=帮|告诉|解释|总结|分析)"
+    replacement: ""
+    saves: 2
+    risk: low
+
+  - id: L1-005
+    layer: L1
+    pattern: "辛苦你?"
+    replacement: ""
+    saves: 2
+    risk: low
+
+  - id: L1-006
+    layer: L1
+    pattern: "(?:谢谢|感谢)(?:你|了)?[!!.。]?"
+    replacement: ""
+    saves: 2
+    risk: low
+
+  # ---- 进行/做 + 动词性名词 → 单字动词 ----
+  - id: L1-010
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?分析"
+    replacement: "分析"
+    saves: 2
+    risk: low
+
+  - id: L1-011
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?总结"
+    replacement: "总结"
+    saves: 2
+    risk: low
+
+  - id: L1-012
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?处理"
+    replacement: "处理"
+    saves: 2
+    risk: low
+
+  - id: L1-013
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?解释"
+    replacement: "解释"
+    saves: 2
+    risk: low
+
+  - id: L1-014
+    layer: L1
+    pattern: "做(?:一?(?:个|下|次)?)?判断"
+    replacement: "判定"
+    saves: 3
+    risk: low
+
+  - id: L1-015
+    layer: L1
+    pattern: "做(?:一?(?:个|下|次)?)?解释"
+    replacement: "解释"
+    saves: 3
+    risk: low
+
+  - id: L1-016
+    layer: L1
+    pattern: "给(?:出|我)(?:一些|几个)?建议"
+    replacement: "建议"
+    saves: 2
+    risk: low
+
+  - id: L1-017
+    layer: L1
+    pattern: "提供(?:一些|相关|相对)?帮助"
+    replacement: "助"
+    saves: 2
+    risk: mid
+
+  - id: L1-018
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?检查"
+    replacement: "检查"
+    saves: 2
+    risk: low
+
+  - id: L1-019
+    layer: L1
+    pattern: "进行(?:一?(?:个|下|次)?)?优化"
+    replacement: "优化"
+    saves: 2
+    risk: low
+
+  # ---- 连接词 ----
+  - id: L1-020
+    layer: L1
+    pattern: "也就是说[,,]?"
+    replacement: "即"
+    saves: 3
+    risk: low
+
+  - id: L1-021
+    layer: L1
+    pattern: "换句话说[,,]?"
+    replacement: "即"
+    saves: 3
+    risk: low
+
+  - id: L1-022
+    layer: L1
+    pattern: "与此同时[,,]?"
+    replacement: "同时,"
+    saves: 2
+    risk: low
+
+  - id: L1-023
+    layer: L1
+    pattern: "在这种情况下[,,]?"
+    replacement: "此时,"
+    saves: 3
+    risk: low
+
+  - id: L1-024
+    layer: L1
+    pattern: "由此可见[,,]?"
+    replacement: "故"
+    saves: 3
+    risk: low
+
+  - id: L1-025
+    layer: L1
+    pattern: "因此(?:[,,]|说)?"
+    replacement: "故"
+    saves: 1
+    risk: low
+
+  - id: L1-026
+    layer: L1
+    pattern: "如果没有"
+    replacement: "若无"
+    saves: 2
+    risk: low
+
+  - id: L1-027
+    layer: L1
+    pattern: "通过(.+?)的方式"
+    replacement: "用\\1"
+    saves: 2
+    risk: mid
+
+  - id: L1-028
+    layer: L1
+    pattern: "(?:如上所述|前面提到的|刚才说的)"
+    replacement: "前述"
+    saves: 3
+    risk: low
+
+  # ---- 修饰副词 ----
+  - id: L1-030
+    layer: L1
+    pattern: "比较(?:简洁|清晰|详细)地?"
+    replacement: ""
+    saves: 3
+    risk: low
+
+  - id: L1-031
+    layer: L1
+    pattern: "相对(?:简洁|详细|完整)地?"
+    replacement: ""
+    saves: 3
+    risk: low
+
+  - id: L1-032
+    layer: L1
+    pattern: "尽可能(?:地)?"
+    replacement: "尽量"
+    saves: 1
+    risk: low
+
+  - id: L1-033
+    layer: L1
+    pattern: "非常(?:详细|详尽|全面)地?"
+    replacement: "详细"
+    saves: 2
+    risk: low
+
+  # ============================================================
+  # L2: 句法层
+  # ============================================================
+  - id: L2-001
+    layer: L2
+    pattern: "对(.+?)进行(?:一?(?:个|下|次)?(?:全面|详细|简要|认真|深入)?的?)?([\\u4e00-\\u9fff]{1,4})"
+    replacement: "\\2\\1"
+    saves: 2
+    risk: mid
+    description: "'对 X 进行 Y' → 'Y X'"
+
+  - id: L2-002
+    layer: L2
+    pattern: "把(.+?)作为(.+?)(?=[,,。.\\s])"
+    replacement: "视\\1为\\2"
+    saves: 2
+    risk: mid
+
+  - id: L2-003
+    layer: L2
+    pattern: "由于(.+?)所以"
+    replacement: "\\1故"
+    saves: 3
+    risk: low
+
+  - id: L2-004
+    layer: L2
+    pattern: "虽然(.+?)但是"
+    replacement: "\\1然"
+    saves: 3
+    risk: mid
+
+  - id: L2-005
+    layer: L2
+    pattern: "不仅(.+?)而且"
+    replacement: "\\1且"
+    saves: 3
+    risk: low
+
+  - id: L2-006
+    layer: L2
+    pattern: "因为(.+?)所以"
+    replacement: "\\1故"
+    saves: 3
+    risk: low
+
+  - id: L2-007
+    layer: L2
+    pattern: "如果(.+?)那么"
+    replacement: "若\\1则"
+    saves: 2
+    risk: low
+
+  # ---- 列表化 ----
+  - id: L2-010
+    layer: L2
+    pattern: "第一[,,]"
+    replacement: "1. "
+    saves: 1
+    risk: low
+
+  - id: L2-011
+    layer: L2
+    pattern: "第二[,,]"
+    replacement: "2. "
+    saves: 1
+    risk: low
+
+  - id: L2-012
+    layer: L2
+    pattern: "第三[,,]"
+    replacement: "3. "
+    saves: 1
+    risk: low
+
+  - id: L2-013
+    layer: L2
+    pattern: "第四[,,]"
+    replacement: "4. "
+    saves: 1
+    risk: low
+
+  - id: L2-014
+    layer: L2
+    pattern: "首先[,,]"
+    replacement: "1. "
+    saves: 1
+    risk: low
+
+  - id: L2-015
+    layer: L2
+    pattern: "其次[,,]"
+    replacement: "2. "
+    saves: 1
+    risk: low
+
+  # ============================================================
+  # L2 协议化重写 (v0.2 修订)
+  # 实测:### 在所有 9 个 tokenizer 上都是 1 token
+  # ============================================================
+  - id: L2-020
+    layer: L2
+    pattern: "请\\s*(?:用|以)?\\s*(?:JSON|json|Json)\\s*格式\\s*(?:输出|返回|回答)"
+    replacement: "\n### 输出\nJSON"
+    saves: 4
+    risk: low
+
+  - id: L2-021
+    layer: L2
+    pattern: "请\\s*(?:用|以)?\\s*中文\\s*(?:回答|回复|输出)"
+    replacement: "\n### 输出\n中文"
+    saves: 3
+    risk: low
+
+  - id: L2-022
+    layer: L2
+    pattern: "请\\s*(?:你)?\\s*扮演\\s*(?:一(?:个|位))?\\s*(.+?)(?=[,,。.\\n]|的角色|$)"
+    replacement: "\n### 角色\n\\1\n"
+    saves: 4
+    risk: high
+    description: |
+      '请你扮演一位 X' → '### 角色\nX'
+      已知问题:含空格的复合 NP 可能被截断,Day 3 用 jieba 修复
+
+  # ============================================================
+  # L3: 成语层(默认 universal 11 条核心成语,需 layer=L3 显式启用)
+  # 在 ≥3 国产 tokenizer 上 1 token,基于 idiom_whitelist.json 实测
+  # ============================================================
+  - id: L3-001
+    layer: L3
+    pattern: "(?:大家都知道|每个人都知道|众人皆知)"
+    replacement: "众所周知"
+    saves: 2
+    risk: mid
+
+  - id: L3-002
+    layer: L3
+    pattern: "投入(?:全部|所有)?(?:精力|力量)(?:去做|做)?"
+    replacement: "全力以赴"
+    saves: 2
+    risk: mid
+
+  - id: L3-003
+    layer: L3
+    pattern: "(?:根据|结合|按照)(?:当地|实际)情况"
+    replacement: "因地制宜"
+    saves: 2
+    risk: mid
+
+  - id: L3-004
+    layer: L3
+    pattern: "(?:一步一步|一步步)(?:地)?(?:推进|进行)"
+    replacement: "循序渐进"
+    saves: 3
+    risk: mid
+
+  - id: L3-005
+    layer: L3
+    pattern: "(?:不断|持续|一直)(?:坚持|努力做)"
+    replacement: "持之以恒"
+    saves: 2
+    risk: mid
+
+  - id: L3-006
+    layer: L3
+    pattern: "认真(?:仔细)?(?:地)?对待"
+    replacement: "脚踏实地"
+    saves: 1
+    risk: mid
+
+  # ============================================================
+  # L4: 协议层归一化
+  # ============================================================
+  - id: L4-001
+    layer: L4
+    pattern: "(?:#+\\s*)?(?:任务|目标|Task|TASK)\\s*[::]\\s*"
+    replacement: "### 任务\n"
+    saves: 0
+    risk: low
+
+  - id: L4-002
+    layer: L4
+    pattern: "(?:#+\\s*)?(?:角色|身份|Role|ROLE)\\s*[::]\\s*"
+    replacement: "### 角色\n"
+    saves: 0
+    risk: low
+
+  - id: L4-003
+    layer: L4
+    pattern: "(?:#+\\s*)?(?:输出|返回|输出格式|Output|OUTPUT)\\s*[::]\\s*"
+    replacement: "### 输出\n"
+    saves: 0
+    risk: low
+
+  - id: L4-004
+    layer: L4
+    pattern: "(?:#+\\s*)?(?:约束|限制|要求|规则|Constraints|CONSTRAINTS)\\s*[::]\\s*"
+    replacement: "### 约束\n"
+    saves: 0
+    risk: low