update

modular_graph_and_candidates.py

@@ -94,10 +94,10 @@ def similarity_clusters(bags: Dict[str, List[Set[str]]], thr: float) -> Dict[Tup
             out[(m1, m2)] = s
     return out
 
-@spaces.GPU
-def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: float) -> Dict[Tuple[str, str], float]:
-    model = SentenceTransformer("codesage/codesage-large-v2", trust_remote_code=True)
-    model.max_seq_length = 4096  # truncate overly long modeling files
+#@spaces.GPU
+def old_embedding_similarity_clusters(models_root: Path, missing: List[str], thr: float) -> Dict[Tuple[str, str], float]:
+    model = SentenceTransformer("codesage/codesage-large-v2", device="cpu", trust_remote_code=True)
+    model.max_seq_length = 8192  # truncate overly long modeling files
     texts = {}
 
     for name in tqdm(missing, desc="Reading modeling files"):
@@ -113,7 +113,7 @@ def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: fl
     all_embeddings = []
 
     print("Encoding embeddings...")
-    batch_size = 8  # or 2 if memory is tight
+    batch_size = 2
     for i in tqdm(range(0, len(names), batch_size), desc="Batches", leave=False):
         batch = [texts[n] for n in names[i:i+batch_size]]
         emb = model.encode(batch, convert_to_numpy=True, show_progress_bar=False)
@@ -122,7 +122,7 @@ def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: fl
     embeddings = np.vstack(all_embeddings)  # [N, D]
 
     print("Computing pairwise similarities...")
-    sims = embeddings @ embeddings.T  # cosine since already normalized
+    sims = embeddings @ embeddings.T
 
     out = {}
     for i in range(len(names)):
@@ -132,6 +132,59 @@ def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: fl
                 out[(names[i], names[j])] = float(s)
     return out
 
+#@spaces.GPU
+def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: float) -> Dict[Tuple[str, str], float]:
+    model = SentenceTransformer("codesage/codesage-large-v2", device="cpu", trust_remote_code=True)
+
+    # Hard-cap by backend max positions (prevents IndexError in self.wpe)
+    try:
+        cfg = model[0].auto_model.config
+        pos_limit = int(getattr(cfg, "n_positions", getattr(cfg, "max_position_embeddings")))
+    except Exception:
+        pos_limit = 1024  # conservative fallback if config is odd
+
+    seq_len = min(pos_limit, 2048)  # optional extra ceiling if pos_limit is huge
+    model.max_seq_length = seq_len               # SentenceTransformer wrapper
+    model[0].max_seq_length = seq_len            # its Transformer submodule actually used for tokenize()
+    model[0].tokenizer.model_max_length = seq_len  # ensure tokenizer truncates
+
+    texts = {}
+    for name in tqdm(missing, desc="Reading modeling files"):
+        code = ""
+        for py in (models_root / name).rglob("modeling_*.py"):
+            try:
+                code += _strip_source(py.read_text(encoding="utf-8")) + "\n"
+            except Exception:
+                continue
+        texts[name] = code.strip() or " "
+
+    names = list(texts)
+    all_embeddings = []
+
+    print("Encoding embeddings...")
+    batch_size = 2
+    for i in tqdm(range(0, len(names), batch_size), desc="Batches", leave=False):
+        batch = [texts[n] for n in names[i:i+batch_size]]
+        emb = model.encode(batch, convert_to_numpy=True, show_progress_bar=False)
+        all_embeddings.append(emb)
+
+    # Cosine similarity requires normalized vectors; SentenceTransformers doesn't always return them normalized
+    import numpy as np
+    embeddings = np.vstack(all_embeddings).astype(np.float32)
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True) + 1e-12
+    embeddings = embeddings / norms
+
+    print("Computing pairwise similarities...")
+    sims_mat = embeddings @ embeddings.T
+
+    out = {}
+    for i in range(len(names)):
+        for j in range(i + 1, len(names)):
+            s = float(sims_mat[i, j])
+            if s >= thr:
+                out[(names[i], names[j])] = s
+    return out
+
 
 
 
@@ -261,34 +314,72 @@ def generate_html(graph: dict) -> str:
 # ────────────────────────────────────────────────────────────────────────────────
 CSS = """
 @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600&display=swap');
-:root { --base: 60px; }
-body { margin:0; font-family:'Inter',Arial,sans-serif; background:transparent; overflow:hidden; }
-svg { width:100vw; height:100vh; }
-.link { stroke:#999; stroke-opacity:.6; }
-.link.cand { stroke:#e63946; stroke-width:2.5; }
-.node-label { fill:#333; pointer-events:none; text-anchor:middle; font-weight:600; }
-.link-label { fill:#555; font-size:10px; pointer-events:none; text-anchor:middle; }
-.node.base path       { fill:#ffbe0b; }
-.node.derived circle  { fill:#1f77b4; }
-.node.cand circle, .node.cand path { fill:#e63946; }
-#legend { position:fixed; top:18px; left:18px; background:rgba(255,255,255,.92); padding:18px 28px;
-          border-radius:10px; border:1.5px solid #bbb; font-size:18px; box-shadow:0 2px 8px rgba(0,0,0,.08); }
+
+:root{
+  --bg:#ffffff;
+  --text:#222222;
+  --muted:#555555;
+  --outline:#ffffff;
+}
+@media (prefers-color-scheme: dark){
+  :root{
+    --bg:#0b0d10;
+    --text:#e8e8e8;
+    --muted:#c8c8c8;
+    --outline:#000000;
+  }
+}
+
+body{ margin:0; font-family:'Inter',Arial,sans-serif; background:var(--bg); overflow:hidden; }
+svg{ width:100vw; height:100vh; }
+
+.link{ stroke:#999; stroke-opacity:.6; }
+.link.cand{ stroke:#e63946; stroke-width:2.5; }
+
+.node-label{
+  fill:var(--text);
+  pointer-events:none;
+  text-anchor:middle;
+  font-weight:600;
+  paint-order:stroke fill;
+  stroke:var(--outline);
+  stroke-width:3px;
+}
+.link-label{
+  fill:var(--muted);
+  pointer-events:none;
+  text-anchor:middle;
+  font-size:10px;
+  paint-order:stroke fill;
+  stroke:var(--bg);
+  stroke-width:2px;
+}
+
+.node.base image{ width:60px; height:60px; transform:translate(-30px,-30px); }
+.node.derived circle{ fill:#1f77b4; }
+.node.cand circle, .node.cand path{ fill:#e63946; }
+
+#legend{
+  position:fixed; top:18px; left:18px;
+  background:rgba(255,255,255,.92);
+  padding:18px 28px; border-radius:10px; border:1.5px solid #bbb;
+  font-size:18px; box-shadow:0 2px 8px rgba(0,0,0,.08);
+}
+@media (prefers-color-scheme: dark){
+  #legend{ background:rgba(20,22,25,.92); color:#e8e8e8; border-color:#444; }
+}
 """
 
 JS = """
-
 function updateVisibility() {
   const show = document.getElementById('toggleRed').checked;
   svg.selectAll('.link.cand').style('display', show ? null : 'none');
   svg.selectAll('.node.cand').style('display', show ? null : 'none');
-  svg.selectAll('.link-label')
-    .filter(d => d.cand)
-    .style('display', show ? null : 'none');
+  svg.selectAll('.link-label').filter(d => d.cand).style('display', show ? null : 'none');
 }
-
 document.getElementById('toggleRed').addEventListener('change', updateVisibility);
 
-
+const HF_LOGO_URI = "__HF_LOGO_DATA_URI__";
 const graph = __GRAPH_DATA__;
 const W = innerWidth, H = innerHeight;
 const svg = d3.select('#dependency').call(d3.zoom().on('zoom', e => g.attr('transform', e.transform)));
@@ -311,17 +402,21 @@ const node = g.selectAll('g.node')
   .attr('class', d => `node ${d.cls}`)
   .call(d3.drag().on('start', dragStart).on('drag', dragged).on('end', dragEnd));
 
-node.filter(d => d.cls==='base').append('image')
-  .attr('xlink:href', 'hf-logo.svg').attr('x', -30).attr('y', -30).attr('width', 60).attr('height', 60);
-node.filter(d => d.cls!=='base').append('circle').attr('r', d => 20*d.sz);
+const baseSel = node.filter(d => d.cls === 'base');
+if (HF_LOGO_URI){
+  baseSel.append('image').attr('href', HF_LOGO_URI);
+}else{
+  baseSel.append('circle').attr('r', d => 22*d.sz).attr('fill', '#ffbe0b');
+}
+node.filter(d => d.cls !== 'base').append('circle').attr('r', d => 20*d.sz);
+
 node.append('text').attr('class','node-label').attr('dy','-2.4em').text(d => d.id);
 
 const sim = d3.forceSimulation(graph.nodes)
-  .force('link', d3.forceLink(graph.links).id(d => d.id).distance(520)) // tighter links
-  .force('charge', d3.forceManyBody().strength(-600))                   // weaker repulsion
+  .force('link', d3.forceLink(graph.links).id(d => d.id).distance(520))
+  .force('charge', d3.forceManyBody().strength(-600))
   .force('center', d3.forceCenter(W / 2, H / 2))
-  .force('collide', d3.forceCollide(d => d.cls === 'base' ? 50 : 50));  // smaller bubble spacing
-
+  .force('collide', d3.forceCollide(d => 50));
 
 sim.on('tick', () => {
   link.attr('x1', d=>d.source.x).attr('y1', d=>d.source.y)