Update index.html

index.html

@@ -42,11 +42,10 @@
       AutoTokenizer,
       AutoModelForCausalLM
     } from "https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.0/dist/transformers.min.js";
-    // make available to the rest of the page
-    window.HF = { env, AutoTokenizer, AutoModelForCausalLM };
 
-    // IMPORTANT: prevent repo IDs being rewritten to /models/, and allow Hub fetches
-    env.localModelPath = null;          // no "/models/" prefix
+    window.HF = { env, AutoTokenizer, AutoModelForCausalLM };
+    // prevent repo IDs being rewritten to /models/, and allow Hub fetches if ever needed
+    env.localModelPath = null;
     env.allowRemoteModels = true;
     env.useBrowserCache  = true;
     env.backends.onnx.wasm.wasmPaths =
@@ -129,10 +128,10 @@
     /* ---------- Backend (no cross-origin isolation) ---------- */
     env.backends.onnx.webgpu = { enabled: false };
     env.backends.onnx.preferredBackend = "wasm";
-    env.backends.onnx.wasm.numThreads = 1;   // single-thread
-    env.backends.onnx.wasm.proxy = false;    // main thread
+    env.backends.onnx.wasm.numThreads = 1;
+    env.backends.onnx.wasm.proxy = false;
     if (typeof env.backends.onnx.wasm.jsep !== "undefined") {
-      env.backends.onnx.wasm.jsep = false;   // avoid threaded.jsep.wasm
+      env.backends.onnx.wasm.jsep = false;
     }
 
     /* ---------- DOM helpers ---------- */
@@ -149,7 +148,7 @@
     const PUNC_ONLY = /^[\s.,;:!?—-]+$/;
 
     /* ---------- Progress UI ---------- */
-    const files = new Map(); // fileURL -> { loaded, total, startedAt, done, cached }
+    const files = new Map();
     let phase = ""; let lastTickBytes = 0, lastTickTime = 0;
     const STALL_SECS = 20;
     const humanMB = (b)=> (b/1024/1024).toFixed(1)+" MB";
@@ -212,8 +211,10 @@
     /* ---------- Model registry ---------- */
     const MODELS = {
       qwen: {
-        local_dir: "assets/models/qwen/",                 // keep for reference
-        local_url: new URL("assets/models/qwen/", window.location.href), // <-- ADD
+        // Local base as a URL object (forces local resolution in transformers.js)
+        local_url: new URL("assets/models/qwen/", window.location.href),
+        // (optional string for reference, not used to load)
+        local_dir: "assets/models/qwen/",
         onnx_file: "onnx/model_q4f16.onnx",
         emb_coords: "assets/embeddings/qwen_pca_top5k_coords.json",
         emb_nbrs:   "assets/embeddings/qwen_neighbors_top5k_k40.json"
@@ -225,7 +226,7 @@
       }
     };
 
-    /* ---------- Qwen3 config shim (treat as Qwen2 layout) ---------- */
+    /* ---------- Qwen3 config shim ---------- */
     const QWEN3_CONFIG_FIX = {
       model_type: "qwen2",
       architectures: ["Qwen2ForCausalLM"]
@@ -304,18 +305,10 @@
       setStatus("Tokenizer: starting…");
       try {
         if (key === "qwen") {
-          // Use strictly local tokenizer first
-          try {
-            tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.local_dir, {
-              progress_callback: onProgress,
-            });
-          } catch (e) {
-            console.warn("Local Qwen tokenizer failed; falling back to Hub:", e);
-            // Hub fallback (now safe: no '/models/' rewrite because localModelPath=null)
-            tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.hub_tokenizer_repo, {
-              progress_callback: onProgress,
-            });
-          }
+          // Use the URL object to force local loading (prevents hub.js path)
+          tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.local_url, {
+            progress_callback: onProgress,
+          });
         } else {
           tokenizer = await AutoTokenizer.from_pretrained(MODELS.distilgpt2.remote_repo, {
             progress_callback: onProgress,
@@ -421,6 +414,7 @@
       let m = -Infinity; for (let i = 0; i < data.length; i++) if (data[i] > m) m = data[i];
       const exps = new Float32Array(data.length); let Z = 0;
       for (let i = 0; i < data.length; i++) { const e = Math.exp(data[i] - m); exps[i] = e; Z += e; }
+
       const K = Math.min(parseInt(topkSel.value, 10) || topK, data.length);
       const idx = Array.from({ length: data.length }, (_, i) => [exps[i] / Z, i])
         .sort((a, b) => b[0] - a[0]).slice(0, K);