Initial Linux build for VIL encoder

.gitattributes

@@ -0,0 +1,2 @@
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+.venv/
+venv/
+.cache/
+.env

README.md

@@ -0,0 +1,21 @@
+---
+title: VIL Encoder
+emoji: ⚙️
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+sdk_version: "4.31.5"
+python_version: "3.10"
+app_file: app.py
+pinned: false
+---
+
+# VIL Encoder Space
+
+Dataset-backed VIL encoder with similarity search and glyphstring sigil render.
+
+## Features
+- Tri-key input: visible / braille / hanzi
+- Learned encoder checkpoint
+- FAISS nearest-neighbor retrieval
+- Deterministic glyphstring sigil render

app.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+import faiss
+import gradio as gr
+import numpy as np
+import torch
+import torch.nn as nn
+
+MODEL_PATH = Path("vil-encoder-v2.pt")
+DATA_PATHS = [
+    Path("data/train.jsonl"),
+    Path("data/validation.jsonl"),
+    Path("data/test.jsonl"),
+]
+DEVICE = "cpu"
+SEQ_LEN = 64
+EMBED_DIM = 32
+
+def encode_triplet(visible: str, braille: str, hanzi: str) -> np.ndarray:
+    text = f"{visible}|{braille}|{hanzi}"
+    arr = np.array([ord(c) % 256 for c in text], dtype=np.float32)
+    if arr.shape[0] < SEQ_LEN:
+        arr = np.pad(arr, (0, SEQ_LEN - arr.shape[0]))
+    else:
+        arr = arr[:SEQ_LEN]
+    arr /= 255.0
+    return arr.astype(np.float32)
+
+class Encoder(nn.Module):
+    def __init__(self, input_dim: int = SEQ_LEN, embed_dim: int = EMBED_DIM) -> None:
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(input_dim, 128),
+            nn.ReLU(),
+            nn.Linear(128, 64),
+            nn.ReLU(),
+            nn.Linear(64, embed_dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        z = self.net(x)
+        return nn.functional.normalize(z, dim=-1)
+
+def load_dataset() -> List[Dict[str, Any]]:
+    rows: List[Dict[str, Any]] = []
+    for p in DATA_PATHS:
+        if p.exists():
+            with p.open("r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        rows.append(json.loads(line))
+    return rows
+
+def load_model() -> tuple[Encoder, Dict[str, Any]]:
+    model = Encoder()
+    status: Dict[str, Any] = {
+        "loaded": False,
+        "model_path": str(MODEL_PATH),
+        "error": None,
+    }
+
+    if not MODEL_PATH.exists():
+        status["error"] = f"missing model: {MODEL_PATH}"
+        return model.eval(), status
+
+    try:
+        obj = torch.load(MODEL_PATH, map_location=DEVICE)
+        if isinstance(obj, dict) and "model_state_dict" in obj:
+            model.load_state_dict(obj["model_state_dict"], strict=True)
+        elif isinstance(obj, dict):
+            model.load_state_dict(obj, strict=False)
+        else:
+            raise RuntimeError(f"unsupported checkpoint type: {type(obj).__name__}")
+        model.eval()
+        status["loaded"] = True
+        return model, status
+    except Exception as e:
+        status["error"] = str(e)
+        model.eval()
+        return model, status
+
+DATASET = load_dataset()
+MODEL, MODEL_STATUS = load_model()
+
+INDEX = faiss.IndexFlatL2(EMBED_DIM)
+EMBED_MATRIX = None
+
+def model_embed(v: str, b: str, h: str) -> np.ndarray:
+    vec = encode_triplet(v, b, h)
+    x = torch.from_numpy(vec).unsqueeze(0)
+    with torch.no_grad():
+        z = MODEL(x).cpu().numpy()[0]
+    return z.astype(np.float32)
+
+def build_index() -> None:
+    global EMBED_MATRIX
+    if not DATASET or not MODEL_STATUS["loaded"]:
+        EMBED_MATRIX = np.zeros((0, EMBED_DIM), dtype=np.float32)
+        return
+    vectors = []
+    for row in DATASET:
+        vectors.append(model_embed(row["visible"], row["braille"], row["hanzi"]))
+    EMBED_MATRIX = np.stack(vectors).astype(np.float32)
+    INDEX.add(EMBED_MATRIX)
+
+build_index()
+
+def render_sigil(v: str, b: str, h: str) -> str:
+    glyphstring = f"{v}{b}{h}"
+    locked = f"⊏⚙{glyphstring}⚙⊐"
+    svg = f"""
+    <svg width="320" height="200" xmlns="http://www.w3.org/2000/svg">
+      <rect width="100%" height="100%" fill="black"/>
+      <text x="50%" y="50%" dominant-baseline="middle" text-anchor="middle"
+            fill="white" font-size="36" font-family="monospace">{locked}</text>
+    </svg>
+    """
+    return svg
+
+def nearest(v: str, b: str, h: str, k: int = 5) -> List[Dict[str, Any]]:
+    if not DATASET or not MODEL_STATUS["loaded"] or INDEX.ntotal == 0:
+        return []
+    q = model_embed(v, b, h).reshape(1, -1)
+    distances, indices = INDEX.search(q, k)
+    out: List[Dict[str, Any]] = []
+    for dist, idx in zip(distances[0].tolist(), indices[0].tolist()):
+        if idx < 0 or idx >= len(DATASET):
+            continue
+        row = dict(DATASET[idx])
+        row["_distance"] = float(dist)
+        out.append(row)
+    return out
+
+def run_pipeline(visible: str, braille: str, hanzi: str):
+    visible = (visible or "").strip()
+    braille = (braille or "").strip()
+    hanzi = (hanzi or "").strip()
+
+    if not visible or not braille or not hanzi:
+        return {"error": "Provide visible, braille, and hanzi."}, ""
+
+    if not MODEL_STATUS["loaded"]:
+        return {"error": "Model not loaded.", "model_status": MODEL_STATUS}, ""
+
+    embedding = model_embed(visible, braille, hanzi).tolist()
+    matches = nearest(visible, braille, hanzi, k=5)
+    svg = render_sigil(visible, braille, hanzi)
+
+    result = {
+        "input": {
+            "visible": visible,
+            "braille": braille,
+            "hanzi": hanzi,
+        },
+        "embedding": embedding,
+        "nearest": matches,
+        "glyphstring": f"{visible}{braille}{hanzi}",
+        "sigil": f"⊏⚙{visible}{braille}{hanzi}⚙⊐",
+    }
+    return result, svg
+
+def search_visible(query: str):
+    query = (query or "").strip()
+    if not query:
+        return []
+    return [row for row in DATASET if query in str(row.get("visible", ""))][:10]
+
+with gr.Blocks(title="VIL Encoder — Glyphmatic Inference Engine") as demo:
+    gr.Markdown("# VIL Encoder — Glyphmatic Inference Engine")
+
+    with gr.Tab("Encode"):
+        visible = gr.Textbox(label="Visible Canon", placeholder="✶")
+        braille = gr.Textbox(label="Invisible Braille", placeholder="⠁")
+        hanzi = gr.Textbox(label="Hanzi Context", placeholder="一")
+        run_btn = gr.Button("Run")
+        result_json = gr.JSON()
+        sigil_svg = gr.HTML()
+
+        run_btn.click(
+            fn=run_pipeline,
+            inputs=[visible, braille, hanzi],
+            outputs=[result_json, sigil_svg],
+        )
+
+    with gr.Tab("Search Dataset"):
+        query = gr.Textbox(label="Query Visible", placeholder="✶")
+        query_btn = gr.Button("Search")
+        query_out = gr.JSON()
+        query_btn.click(fn=search_visible, inputs=[query], outputs=[query_out])
+
+    with gr.Tab("System Info"):
+        gr.JSON(
+            {
+                "device": DEVICE,
+                "model_status": MODEL_STATUS,
+                "dataset_rows": len(DATASET),
+                "index_size": int(INDEX.ntotal),
+            }
+        )
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)

requirements.txt

@@ -0,0 +1,6 @@
+gradio==4.31.5
+huggingface_hub==0.23.0
+torch==2.3.1
+numpy==1.26.4
+pandas==2.2.2
+faiss-cpu==1.8.0

train_vil_encoder_v2.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+from pathlib import Path
+from typing import List, Tuple
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import Dataset, DataLoader
+
+TRAIN_PATH = Path("data/train.jsonl")
+MODEL_OUT = Path("vil-encoder-v2.pt")
+
+SEQ_LEN = 64
+EMBED_DIM = 32
+BATCH_SIZE = 128
+EPOCHS = 12
+LR = 1e-3
+WEIGHT_DECAY = 1e-5
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+SEED = 918
+
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+def encode_triplet(visible: str, braille: str, hanzi: str) -> np.ndarray:
+    text = f"{visible}|{braille}|{hanzi}"
+    arr = np.array([ord(c) % 256 for c in text], dtype=np.float32)
+    if arr.shape[0] < SEQ_LEN:
+        arr = np.pad(arr, (0, SEQ_LEN - arr.shape[0]))
+    else:
+        arr = arr[:SEQ_LEN]
+    arr /= 255.0
+    return arr
+
+def load_rows(path: Path) -> List[dict]:
+    rows: List[dict] = []
+    with path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    if not rows:
+        raise RuntimeError(f"No rows loaded from {path}")
+    return rows
+
+class PairDataset(Dataset):
+    def __init__(self, rows: List[dict]) -> None:
+        self.rows = rows
+        self.inputs = np.stack([
+            encode_triplet(r["visible"], r["braille"], r["hanzi"]) for r in rows
+        ]).astype(np.float32)
+
+    def __len__(self) -> int:
+        return len(self.rows)
+
+    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
+        anchor = self.inputs[idx]
+        pos_idx = (idx + 1) % len(self.inputs)
+        positive = self.inputs[pos_idx]
+        return torch.from_numpy(anchor), torch.from_numpy(positive)
+
+class Encoder(nn.Module):
+    def __init__(self, input_dim: int = SEQ_LEN, embed_dim: int = EMBED_DIM) -> None:
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(input_dim, 128),
+            nn.ReLU(),
+            nn.Linear(128, 64),
+            nn.ReLU(),
+            nn.Linear(64, embed_dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        z = self.net(x)
+        return nn.functional.normalize(z, dim=-1)
+
+def cosine_pull_loss(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
+    return 1.0 - nn.functional.cosine_similarity(a, b).mean()
+
+def main() -> None:
+    rows = load_rows(TRAIN_PATH)
+    dataset = PairDataset(rows)
+    loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, drop_last=False)
+
+    model = Encoder().to(DEVICE)
+    optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)
+
+    best_loss = float("inf")
+    history = []
+
+    for epoch in range(EPOCHS):
+        model.train()
+        running = 0.0
+        batches = 0
+
+        for x1, x2 in loader:
+            x1 = x1.to(DEVICE)
+            x2 = x2.to(DEVICE)
+
+            z1 = model(x1)
+            z2 = model(x2)
+
+            loss = cosine_pull_loss(z1, z2)
+
+            optimizer.zero_grad(set_to_none=True)
+            loss.backward()
+            optimizer.step()
+
+            running += float(loss.item())
+            batches += 1
+
+        epoch_loss = running / max(1, batches)
+        history.append(epoch_loss)
+        print(f"epoch={epoch:02d} loss={epoch_loss:.6f}")
+
+        if epoch_loss < best_loss:
+            best_loss = epoch_loss
+            checkpoint = {
+                "model_state_dict": model.state_dict(),
+                "config": {
+                    "input_dim": SEQ_LEN,
+                    "embed_dim": EMBED_DIM,
+                },
+                "history": history,
+            }
+            torch.save(checkpoint, MODEL_OUT)
+
+    print(f"saved={MODEL_OUT} best_loss={best_loss:.6f}")
+
+if __name__ == "__main__":
+    main()

vil-encoder-v2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e193779c9a2db196bc71bdd470cc22b5a8694d2fcd294fc7a986762cfd6fe6d9
+size 77646

vil_dataset_builder.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import json
+import hashlib
+import random
+from pathlib import Path
+
+OUT_DIR = Path("data")
+SEED = 918
+random.seed(SEED)
+
+TRAIN_SIZE = 5000
+VAL_SIZE = 1000
+TEST_SIZE = 1000
+
+VISIBLE_GLYPHS = [
+    "✶","✷","✸","✹","✺","✻","✼","✽",
+    "✾","✿","❀","❁","❂","❃","❄","❅"
+]
+
+BRAILLE_STATES = [
+    "⠁","⠃","⠇","⠏","⠟","⠿","⡇","⡿",
+    "⡟","⡯","⡷","⡻","⠻","⠽","⠷","⢿"
+]
+
+HANZI_CONTEXT = [
+    "一","二","三","四","五","六","七","八",
+    "九","十","百","千","万","亿","兆","世"
+]
+
+def compute_digest(visible: str, braille: str, hanzi: str) -> str:
+    payload = f"{visible}|{braille}|{hanzi}".encode("utf-8")
+    return hashlib.sha3_256(payload).hexdigest()
+
+def semantic_weight(visible: str, braille: str, hanzi: str) -> float:
+    v = VISIBLE_GLYPHS.index(visible) / max(1, len(VISIBLE_GLYPHS) - 1)
+    b = BRAILLE_STATES.index(braille) / max(1, len(BRAILLE_STATES) - 1)
+    h = HANZI_CONTEXT.index(hanzi) / max(1, len(HANZI_CONTEXT) - 1)
+    return round(0.4 * v + 0.3 * b + 0.3 * h, 6)
+
+def generate_row(idx: int) -> dict:
+    visible = random.choice(VISIBLE_GLYPHS)
+    braille = random.choice(BRAILLE_STATES)
+    hanzi = random.choice(HANZI_CONTEXT)
+    return {
+        "glyph_id": f"glyph_{idx:08d}",
+        "visible": visible,
+        "braille": braille,
+        "hanzi": hanzi,
+        "semantic_weight": semantic_weight(visible, braille, hanzi),
+        "digest": compute_digest(visible, braille, hanzi),
+        "tri_key": {
+            "visible_layer": visible,
+            "state_layer": braille,
+            "context_layer": hanzi,
+        },
+    }
+
+def write_split(path: Path, start: int, size: int) -> None:
+    with path.open("w", encoding="utf-8") as f:
+        for i in range(size):
+            row = generate_row(start + i)
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+def main() -> None:
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+    write_split(OUT_DIR / "train.jsonl", 0, TRAIN_SIZE)
+    write_split(OUT_DIR / "validation.jsonl", TRAIN_SIZE, VAL_SIZE)
+    write_split(OUT_DIR / "test.jsonl", TRAIN_SIZE + VAL_SIZE, TEST_SIZE)
+    print("Built dataset at ./data")
+
+if __name__ == "__main__":
+    main()