Update index.html

**fix: patch tokenizer load for Qwen**

* Use local tokenizer path first; fall back to Hub tokenizer if local fails
* Prevent `t.replace is not a function` crash during `from_pretrained`
* Keep Qwen model loading local; only tokenizer may use Hub fallback

index.html

@@ -114,13 +114,12 @@
       </section>
     </div>
   </main>
-
   <script type="module">
     const { env, AutoTokenizer, AutoModelForCausalLM } = window.HF;
-
+  
     // Return a URL object pointing to an app-relative path
     const ABS = (p) => new URL(p, window.location.href);
-
+  
     /* ---------- ONNX Runtime Web backend selection (compat mode) ---------- */
     env.backends.onnx.webgpu = { enabled: false };   // disable WebGPU
     env.backends.onnx.preferredBackend = "wasm";
@@ -131,7 +130,7 @@
     }
     env.backends.onnx.wasm.wasmPaths =
       "https://cdn.jsdelivr.net/npm/onnxruntime-web@1.17.1/dist/";
-
+  
     /* ---------- DOM helpers ---------- */
     const $ = (s) => document.querySelector(s);
     const statusEl = $('#status'), barEl = $('#bar'), errEl = $('#error');
@@ -144,17 +143,17 @@
     function setErr(e){ errEl.textContent = e || ""; }
     function showToken(s){ if (s === "\n") return "⏎"; if (s.trim() === "") return `␣${s.length>1 ? "×"+s.length : ""}`; return s; }
     const PUNC_ONLY = /^[\s.,;:!?—-]+$/;
-
+  
     /* ---------- Byte-accurate progress with speed + ETA ---------- */
     const files = new Map(); // fileURL -> { loaded, total, startedAt, done, cached }
     let phase = "";          // "Tokenizer" | "Model"
     let lastTickBytes = 0, lastTickTime = 0;
     const STALL_SECS = 20;
-
+  
     function humanMB(b){ return (b/1024/1024).toFixed(1) + " MB"; }
     function humanSpeed(bps){ if (!bps || !isFinite(bps)) return ""; const mbps=bps/1024/1024; return (mbps>=1?mbps.toFixed(1)+" MB/s":(bps/1024).toFixed(0)+" kB/s"); }
     function humanETA(eta){ return eta>0 ? Math.max(1, Math.round(eta))+"s" : ""; }
-
+  
     function resetProgress(nextPhase){
       phase = nextPhase || "";
       files.clear();
@@ -163,7 +162,7 @@
       setStatus(`${phase ? phase+": " : ""}Starting…`);
       setErr("");
     }
-
+  
     // transformers.js will call this for every file chunk
     function onProgress(evt){
       if (evt.file) {
@@ -173,7 +172,7 @@
         if (evt.status === "ready") { f.done = true; if (f.total === 0) f.cached = true; }
         files.set(evt.file, f);
       }
-
+  
       let loadedSum = 0, totalSum = 0, anyTotals = false, allDone = true, anyCached = false;
       for (const f of files.values()){
         loadedSum += f.loaded || 0;
@@ -182,18 +181,18 @@
         allDone   &&= !!f.done;
         anyCached ||= f.cached;
       }
-
+  
       const now = performance.now();
       const dt  = Math.max(1, now - lastTickTime) / 1000;
       const dBytes = loadedSum - lastTickBytes;
       const bps = dBytes / dt;
       lastTickBytes = loadedSum; lastTickTime = now;
-
+  
       if (evt.status === "downloading" || (!evt.status && anyTotals)) {
         let pct = 0;
         if (totalSum > 0) pct = Math.min(100, Math.floor((loadedSum / totalSum) * 100));
         if (barEl) barEl.style.width = (totalSum>0 ? pct : 10) + "%";
-
+  
         const parts = [
           phase ? `${phase}:` : "Downloading…",
           anyTotals ? `${humanMB(loadedSum)} / ${humanMB(totalSum)} (${pct}%)` : `${humanMB(loadedSum)}…`,
@@ -206,13 +205,13 @@
         if (anyCached) parts.push("(cached)");
         setStatus(parts.filter(Boolean).join("  "));
       }
-
+  
       if (evt.status === "ready" || allDone) {
         if (barEl) barEl.style.width = "100%";
         setStatus(`${phase ? phase+": " : ""}Ready`);
       }
     }
-
+  
     // Stall detector (UI hint)
     setInterval(() => {
       const now = performance.now();
@@ -222,17 +221,20 @@
         setErr("Download seems stalled. If you use ad/tracker blockers, disable them for this page or try the smaller model.");
       }
     }, 4000);
-
+  
     /* ---------- Model registry ---------- */
     const MODELS = {
       qwen: {
-        // Load everything locally (no Hub fetch)
+        // Local ONNX graph
         local: ABS("assets/models/qwen/"),
         file_name: "onnx/model_q4f16.onnx",
+        // Embeddings (UI only)
         emb: {
           coords: ABS("assets/embeddings/qwen_pca_top5k_coords.json"),
           nbrs:   ABS("assets/embeddings/qwen_neighbors_top5k_k40.json")
-        }
+        },
+        // Fallback tokenizer on HF Hub (compat with Qwen2 tokenizer)
+        hub_tokenizer: "Qwen/Qwen2.5-0.5B"
       },
       distilgpt2: {
         local: ABS("assets/models/distilgpt2/"),
@@ -244,13 +246,13 @@
         }
       }
     };
-
+  
     /* ---------- Qwen3 config shim (treat as Qwen2 in JS) ---------- */
     const QWEN3_CONFIG_FIX = {
       model_type: "qwen2",
       architectures: ["Qwen2ForCausalLM"]
     };
-
+  
     /* ---------- Embedding viewer ---------- */
     const Emb = (() => {
       let coordsPath = "", nbrsPath = "";
@@ -284,42 +286,52 @@
       function bounds(){ let xmin=Infinity,xmax=-Infinity,ymin=Infinity,ymax=-Infinity; for(const p of points){ if(p.x<xmin)xmin=p.x; if(p.x>xmax)xmax=p.x; if(p.y<ymin)ymin=p.y; if(p.y>ymax)ymax=p.y; } if(!isFinite(xmin)){xmin=0;xmax=1;ymin=0;ymax=1;} return {xmin,xmax,ymin,ymax}; }
       function toCanvas(x,y){ const pad=24,w=embCanvas.width,h=embCanvas.height; const {xmin,xmax,ymin,ymax}=bounds(); const tx=pad+(x-xmin)/Math.max(1e-9,(xmax-xmin))*(w-pad*2); const ty=pad+(1-(y-ymin)/Math.max(1e-9,(ymax-ymin)))*(h-pad*2); return [tx,ty]; }
       function drawBase(){ const ctx=embCtx; ctx.clearRect(0,0,embCanvas.width,embCanvas.height); ctx.fillStyle="#0b1327"; ctx.fillRect(0,0,embCanvas.width,embCanvas.height); ctx.fillStyle="#5f7aa5"; for(const p of points){ const [x,y]=toCanvas(p.x,p.y); ctx.fillRect(x,y,2,2); } }
-      function highlight(token){ if(!baseDrawn) drawBase(); const ctx=embCtx; const base=index.get(token); if(!base) return; const nbrs=neighbors.get(token)||[]; ctx.strokeStyle="#38bdf8"; ctx.lineWidth=1; const [bx,by]=toCanvas(base.x,base.y); for(const t of nbrs){ const p=index.get(t); if(!p) continue; const [x,y]=toCanvas(p.x,p.y); ctx.beginPath(); ctx.moveTo(bx,by); ctx.lineTo(x,y); ctx.stroke(); ctx.fillStyle="#9bd7ff"; ctx.fillRect(x-2,y-2,4,4);} ctx.fillStyle="#ffd166"; ctx.beginPath(); ctx.arc(bx,by,5,0,Math.PI*2); ctx.fill(); ctx.fillStyle="#e6f1ff"; ctx.font="12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"; ctx.fillText(showToken(token), bx+8, by-8); }
+      function highlight(token){ if(!baseDrawn) drawBase(); const ctx=embCtx; const base=index.get(token); if(!base) return; const nbrs=neighbors.get(token)||[]; ctx.strokeStyle="#38bdf8"; ctx.lineWidth=1; const [bx,by]=toCanvas(base.x,base.y); for(const t of nbrs){ const p=index.get(t); if(!p) continue; const [x,y]=toCanvas(p.x,p.y); ctx.beginPath(); ctx.moveTo(bx,by); ctx.lineTo(x,y); ctx.stroke(); ctx.fillStyle="#9bd7ff"; ctx.fillRect(x-2,y-2,4-0,4); } // small dot
+      ctx.fillStyle="#ffd166"; ctx.beginPath(); ctx.arc(bx,by,5,0,Math.PI*2); ctx.fill(); ctx.fillStyle="#e6f1ff"; ctx.font="12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace"; ctx.fillText(showToken(token), bx+8, by-8); }
       return { setSources, load, drawBase, highlight };
     })();
-
+  
     /* ---------- Core model state ---------- */
     let tokenizer=null, model=null;
     let loadSeq = 0;
-
+  
     async function loadModel(key){
       const mySeq = ++loadSeq;
-
+  
       // Embeddings (unchanged)
       Emb.setSources(key);
       try { await Emb.load(); } catch { embStatus.textContent = "Map failed to load"; }
-
+  
       // --- Tokenizer ---
       resetProgress("Tokenizer");
       setStatus("Tokenizer: starting…");
       try {
-        // IMPORTANT:
-        // - For local Qwen: pass the URL object directly (prevents HF prefixing).
-        // - For distilgpt2 (Hub): pass the repo id string.
-        const tokBase = (key === "qwen")
-          ? MODELS.qwen.local                // URL object (local directory)
-          : MODELS[key].remote;              // "Xenova/distilgpt2"
-
-        tokenizer = await AutoTokenizer.from_pretrained(tokBase, {
-          progress_callback: onProgress,
-        });
+        if (key === "qwen") {
+          // Try local tokenizer first (URL object keeps it local)
+          try {
+            tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.local, {
+              progress_callback: onProgress,
+            });
+          } catch (e) {
+            console.warn("Local Qwen tokenizer failed; falling back to Hub:", e);
+            // Fallback to a Hub tokenizer compatible with Qwen2 vocab
+            tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.hub_tokenizer, {
+              progress_callback: onProgress,
+            });
+          }
+        } else {
+          // distilgpt2: use Hub repo id (string)
+          tokenizer = await AutoTokenizer.from_pretrained(MODELS[key].remote, {
+            progress_callback: onProgress,
+          });
+        }
       } catch (e) {
         console.error("Tokenizer load failed:", e);
         setErr("Tokenizer failed to load.");
         return;
       }
       if (mySeq !== loadSeq) return;
-
+  
       // --- Model ---
       resetProgress("Model");
       setStatus("Model: starting…");
@@ -342,7 +354,7 @@
         return;
       }
       if (mySeq !== loadSeq) return;
-
+  
       // --- Warm-up (guarded) ---
       setStatus("Warming up…");
       try {
@@ -359,20 +371,20 @@
       if (mySeq !== loadSeq) return;
       setStatus("Ready");
     }
-
+  
     /* ---------- Next-token logic ---------- */
     async function greedyNext(text, topK = 10) {
       if (!tokenizer || !model) {
         setErr("Model not loaded yet — check the status bar.");
         return { rows: [], dt: 0 };
       }
-
+  
       // 1) Tokenize
       let enc = await tokenizer(text ?? " ", {
         add_special_tokens: false,
         return_attention_mask: true
       });
-
+  
       // retry with BOS if empty
       const len = enc?.input_ids?.dims?.at(-1) ?? 0;
       if (!len || len <= 0) {
@@ -381,10 +393,10 @@
           return_attention_mask: true
         });
       }
-
+  
       const eosId = tokenizer?.eos_token_id ?? model?.config?.eos_token_id ?? undefined;
       const padId = tokenizer?.pad_token_id ?? eosId;
-
+  
       const t0 = performance.now();
       let gen;
       try {
@@ -411,18 +423,18 @@
         });
       }
       const dt = (performance.now() - t0) | 0;
-
+  
       // logits -> softmax -> Top-K
       const logitsT = gen.scores[0];
       const data = logitsT.data;
       let m = -Infinity; for (let i = 0; i < data.length; i++) if (data[i] > m) m = data[i];
       const exps = new Float32Array(data.length); let Z = 0;
       for (let i = 0; i < data.length; i++) { const e = Math.exp(data[i] - m); exps[i] = e; Z += e; }
-
+  
       const K = Math.min(parseInt(topkSel.value, 10) || topK, data.length);
       const idx = Array.from({ length: data.length }, (_, i) => [exps[i] / Z, i])
         .sort((a, b) => b[0] - a[0]).slice(0, K);
-
+  
       const rows = [];
       for (const [p, i] of idx) {
         const tok = await tokenizer.decode([i], { skip_special_tokens: false });
@@ -430,7 +442,7 @@
       }
       return { rows, dt };
     }
-
+  
     function renderRows(rows){
       klistEl.innerHTML = "";
       const hide = hidePunc.checked;
@@ -445,7 +457,7 @@
         klistEl.appendChild(row);
       }
     }
-
+  
     async function predict(){
       try{
         setErr(""); predictBtn.disabled = true;
@@ -459,7 +471,7 @@
         predictBtn.disabled = false;
       }
     }
-
+  
     /* ---------- UI ---------- */
     predictBtn.addEventListener('click', predict);
     textEl.addEventListener('input', (() => { let to; return () => { clearTimeout(to); to = setTimeout(predict, 250); }; })());
@@ -467,10 +479,10 @@
     topkSel.addEventListener('change', predict);
     demoBtn.addEventListener('click', () => { textEl.value = "Twinkle, twinkle, little "; predict(); });
     modelSel.addEventListener('change', async (e) => { await loadModel(e.target.value); predict(); });
-
+  
     /* ---------- Boot ---------- */
     (async function init(){
-      await loadModel(modelSel.value);   // defaults to 'qwen' (Qwen3-0.6B)
+      await loadModel(modelSel.value);   // defaults to 'qwen'
       if (!textEl.value) textEl.value = "Twinkle, twinkle, little ";
       await predict();
     })();