Update index.html

index.html

@@ -205,6 +205,7 @@
     const MODELS = {
       qwen: {
         id: "qwen",                         // this is the folder name under assets/models/
+        base: new URL("assets/models/qwen/", window.location.href).href,
         onnx_file: "onnx/model_q4f16.onnx",
         emb_coords: "assets/embeddings/qwen_pca_top5k_coords.json",
         emb_nbrs:   "assets/embeddings/qwen_neighbors_top5k_k40.json",
@@ -274,28 +275,52 @@
       return { setSources, load, drawBase, highlight };
     })();
     
-    // --- Local Qwen tokenizer loader (no Hub, no path rewrite) ---
+    // --- Local Qwen tokenizer loader (supports both JSON and BPE files) ---
     async function loadTokenizerQwenLocal() {
       resetProgress("Tokenizer");
       setStatus("Tokenizer: starting…");
     
-      const baseHref = new URL(MODELS.qwen.base, window.location.href).href;
+      const base = MODELS.qwen.base;              // e.g. .../assets/models/qwen/
+      // New-format filenames
+      const tjson = new URL("tokenizer.json",        base).href;
+      const tcfg  = new URL("tokenizer_config.json", base).href;
+      const smap  = new URL("special_tokens_map.json", base).href;
     
-      const tjsonURL = new URL("tokenizer.json", baseHref).href;
-      const tcfgURL  = new URL("tokenizer_config.json", baseHref).href;
-      const smapURL  = new URL("special_tokens_map.json", baseHref).href; // optional
+      // Classic BPE filenames (present in many ONNX repos, incl. Qwen)
+      const vocab = new URL("vocab.json", base).href;
+      const merges = new URL("merges.txt", base).href;
     
-      const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
-        fetch(tjsonURL).then(r => { if (!r.ok) throw new Error("missing tokenizer.json");        return r.json(); }),
-        fetch(tcfgURL).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json");  return r.json(); }),
-        fetch(smapURL).then(r => r.ok ? r.json() : null),
-      ]);
+      // Try new-format first; if missing, fall back to vocab/merges
+      const tjsonHead = await fetch(tjson, { method: "HEAD" });
+      const useNewFormat = tjsonHead.ok;
     
       const files = new Map();
-      files.set("tokenizer.json",        new Blob([JSON.stringify(tokJSON)], { type: "application/json" }));
-      files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" }));
-      if (smapJSON) {
-        files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
+    
+      if (useNewFormat) {
+        const [tokJSON, tokCfgJSON, smapJSON] = await Promise.all([
+          fetch(tjson).then(r => { if (!r.ok) throw new Error("missing tokenizer.json"); return r.json(); }),
+          fetch(tcfg).then(r => { if (!r.ok) throw new Error("missing tokenizer_config.json"); return r.json(); }),
+          fetch(smap).then(r => r.ok ? r.json() : null),
+        ]);
+        files.set("tokenizer.json",        new Blob([JSON.stringify(tokJSON)],   { type: "application/json" }));
+        files.set("tokenizer_config.json", new Blob([JSON.stringify(tokCfgJSON)],{ type: "application/json" }));
+        if (smapJSON) files.set("special_tokens_map.json", new Blob([JSON.stringify(smapJSON)], { type: "application/json" }));
+      } else {
+        // Fall back to BPE pair; this avoids the "e.split is not a function" crash
+        const [vocabText, mergesText, tokCfgJSON, smapJSON] = await Promise.all([
+          fetch(vocab).then(r => { if (!r.ok) throw new Error("missing vocab.json");  return r.text(); }),
+          fetch(merges).then(r => { if (!r.ok) throw new Error("missing merges.txt"); return r.text(); }),
+          fetch(tcfg).then(r => r.ok ? r.json() : { model_max_length: 32768 }),
+          fetch(smap).then(r => r.ok ? r.json() : null),
+        ]);
+        files.set("vocab.json",  new Blob([vocabText],  { type: "application/json" }));
+        files.set("merges.txt",  new Blob([mergesText], { type: "text/plain" }));
+        files.set("tokenizer_config.json",
+          new Blob([JSON.stringify(tokCfgJSON)], { type: "application/json" })
+        );
+        if (smapJSON) files.set("special_tokens_map.json",
+          new Blob([JSON.stringify(smapJSON)], { type: "application/json" })
+        );
       }
     
       tokenizer = await AutoTokenizer.from_pretrained("", {
@@ -304,10 +329,7 @@
         progress_callback: onProgress,
       });
     }
-    
-    // --- Core model state ---
-    let tokenizer = null, model = null;
-    let loadSeq = 0;
+
 
     // Core model state (module scope)
     let tokenizer = null, model = null;
@@ -325,13 +347,10 @@
       setStatus("Tokenizer: starting…");
       try {
         if (key === "qwen") {
-          env.allowRemoteModels = false; // force local for Qwen
-          tokenizer = await AutoTokenizer.from_pretrained(MODELS.qwen.id, {
-            progress_callback: onProgress,
-            // local load uses env.localModelPath + "qwen"/files
-          });
+          env.allowRemoteModels = false;         // force local for Qwen
+          await loadTokenizerQwenLocal();        // <-- use the local loader
         } else {
-          env.allowRemoteModels = true;  // hub is fine for distilgpt2
+          env.allowRemoteModels = true;          // Hub is fine for distilgpt2
           tokenizer = await AutoTokenizer.from_pretrained(MODELS.distilgpt2.repo, {
             progress_callback: onProgress,
           });
@@ -341,6 +360,7 @@
         setErr("Tokenizer failed to load.");
         return;
       }
+
       if (mySeq !== loadSeq) return;
       
       // --- Model ---