Update index.html

fix(tokenizer): include merges.txt + vocab.json for Qwen BPE; load as text/JSON in loadTokenizerQwenLocal to avoid e.split error

index.html

@@ -274,6 +274,41 @@
       return { setSources, load, drawBase, highlight };
     })();
     
+    // --- Local Qwen tokenizer loader (no Hub, no path rewrite) ---
+    async function loadTokenizerQwenLocal() {
+      resetProgress("Tokenizer");
+      setStatus("Tokenizer: starting…");
+    
+      const baseHref = new URL(MODELS.qwen.base, window.location.href).href;
+    
+      const tjsonURL = new URL("tokenizer.json", baseHref).href;
+      const tcfgURL  = new URL("tokenizer_config.json", baseHref).href;
+      const smapURL  = new URL("special_tokens_map.json", baseHref).href; // optional
+    
+      const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
+        fetch(tjsonURL).then(r => { if (!r.ok) throw new Error("missing tokenizer.json");        return r.json(); }),
+        fetch(tcfgURL).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json");  return r.json(); }),
+        fetch(smapURL).then(r => r.ok ? r.json() : null),
+      ]);
+    
+      const files = new Map();
+      files.set("tokenizer.json",        new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
+      files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
+      if (smapJSON) {
+        files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
+      }
+    
+      tokenizer = await AutoTokenizer.from_pretrained("", {
+        files,
+        local_files_only: true,
+        progress_callback: onProgress,
+      });
+    }
+    
+    // --- Core model state ---
+    let tokenizer = null, model = null;
+    let loadSeq = 0;
+
     // Core model state (module scope)
     let tokenizer = null, model = null;
     let loadSeq = 0;