Spaces:

quickgrid
/

Tokenizer-Visualizer

Running

App Files Files Community

quickgrid commited on 5 days ago

Commit

36753eb

verified ·

1 Parent(s): fa23c05

Update index.html

Browse files

Files changed (1) hide show

index.html +455 -239

index.html CHANGED Viewed

@@ -1,249 +1,465 @@
 <!DOCTYPE html>
-<html lang="en">
 <head>
-  <meta charset="UTF-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Universal Tokenizer Visualizer</title>
-  <script src="https://cdn.tailwindcss.com"></script>
-  <link href="https://cdn.jsdelivr.net/npm/@fontsource/inter@5.0.16/index.min.css" rel="stylesheet">
-  <style>
-    body { font-family: 'Inter', sans-serif; background-color: #0f172a; }
-    .token-chip {
-      transition: all 0.15s cubic-bezier(0.4, 0, 0.2, 1);
-      cursor: default;
-      position: relative;
-    }
-    .token-chip:hover {
-      transform: translateY(-2px) scale(1.03);
-      z-index: 20;
-      box-shadow: 0 8px 16px -4px rgba(0,0,0,0.5);
-    }
-    .token-chip::after {
-      content: attr(data-tooltip);
-      position: absolute;
-      bottom: 110%;
-      left: 50%;
-      transform: translateX(-50%) translateY(5px);
-      background: #1e293b;
-      color: #e2e8f0;
-      padding: 6px 10px;
-      border-radius: 6px;
-      font-size: 0.75rem;
-      white-space: nowrap;
-      opacity: 0;
-      pointer-events: none;
-      transition: all 0.2s;
-      border: 1px solid #334155;
-      box-shadow: 0 4px 12px rgba(0,0,0,0.3);
-      z-index: 30;
-    }
-    .token-chip:hover::after { opacity: 1; transform: translateX(-50%) translateY(0); }
-    .spinner {
-      width: 20px; height: 20px;
-      border: 2px solid rgba(255,255,255,0.3);
-      border-radius: 50%;
-      border-top-color: #fff;
-      animation: spin 0.8s linear infinite;
-    }
-    @keyframes spin { to { transform: rotate(360deg); } }
-    ::-webkit-scrollbar { width: 8px; height: 8px; }
-    ::-webkit-scrollbar-track { background: #1e293b; border-radius: 4px; }
-    ::-webkit-scrollbar-thumb { background: #475569; border-radius: 4px; }
-    ::-webkit-scrollbar-thumb:hover { background: #64748b; }
-    .fade-in { animation: fadeIn 0.3s ease-out forwards; }
-    @keyframes fadeIn { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } }
-  </style>
 </head>
-<body class="min-h-screen text-slate-100 flex flex-col">
-  <div class="flex-grow max-w-6xl w-full mx-auto p-4 md:p-8">
-    <header class="mb-8 flex flex-col md:flex-row md:items-end justify-between gap-4">
-      <div>
-        <h1 class="text-3xl md:text-4xl font-bold bg-gradient-to-r from-cyan-400 via-blue-500 to-purple-500 bg-clip-text text-transparent tracking-tight">
-          🧩 Tokenizer Visualizer
-        </h1>
-        <p class="text-slate-400 mt-2 text-sm md:text-base">
-          Load any HuggingFace model tokenizer directly in your browser. Client-side only, no GPU required.
-        </p>
-      </div>
-      <div class="flex gap-2 text-xs text-slate-500">
-        <span class="px-2 py-1 bg-slate-800 rounded border border-slate-700">Browser-Native</span>
-        <span class="px-2 py-1 bg-slate-800 rounded border border-slate-700">Extensible</span>
-      </div>
-    </header>
-    <div class="grid grid-cols-1 lg:grid-cols-12 gap-6">
-      <!-- Controls Panel -->
-      <div class="lg:col-span-4 bg-slate-800/50 backdrop-blur border border-slate-700 rounded-xl p-5 flex flex-col gap-4 h-fit shadow-xl">
-        <div>
-          <label class="block text-sm font-medium text-slate-300 mb-1.5">Model ID / Path</label>
-          <input type="text" id="modelId" value="gpt2"
-            class="w-full bg-slate-900 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 transition"
-            placeholder="e.g., gpt2, deepseek-ai/DeepSeek-V3, meta-llama/Llama-3-8B">
-        </div>
-        <div class="flex-grow">
-          <label class="block text-sm font-medium text-slate-300 mb-1.5">Input Text</label>
-          <textarea id="inputText" rows="7"
-            class="w-full bg-slate-900 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 resize-y transition"
-            placeholder="Type or paste text to visualize tokenization..."></textarea>
-        </div>
-        <button id="tokenizeBtn"
-          class="w-full bg-blue-600 hover:bg-blue-500 active:bg-blue-700 text-white font-semibold py-2.5 px-4 rounded-lg transition flex items-center justify-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed shadow-lg shadow-blue-900/30">
-          <span>Tokenize</span>
-        </button>
-        <div id="status" class="text-sm text-slate-400 h-5 flex items-center gap-2 truncate"></div>
-      </div>
-      <!-- Visualization Panel -->
-      <div class="lg:col-span-8 bg-slate-800/50 backdrop-blur border border-slate-700 rounded-xl p-5 flex flex-col shadow-xl min-h-[450px]">
-        <div class="flex flex-wrap justify-between items-center mb-4 pb-3 border-b border-slate-700 gap-3">
-          <h2 class="text-lg font-semibold text-slate-100">Token Breakdown</h2>
-          <div class="flex flex-wrap gap-3 text-sm">
-            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
-              <span class="w-2 h-2 rounded-full bg-blue-400"></span>
-              <span class="text-slate-300">Tokens: <b id="statTokens" class="text-white">0</b></span>
             </div>
-            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
-              <span class="w-2 h-2 rounded-full bg-purple-400"></span>
-              <span class="text-slate-300">Chars: <b id="statChars" class="text-white">0</b></span>
             </div>
-            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
-              <span class="w-2 h-2 rounded-full bg-pink-400"></span>
-              <span class="text-slate-300">Ratio: <b id="statRatio" class="text-white">0.00</b></span>
             </div>
-          </div>
-        </div>
-        <div id="outputGrid" class="flex flex-wrap gap-2 overflow-y-auto p-2 content-start flex-grow min-h-[300px] bg-slate-900/40 rounded-lg border border-slate-800/50">
-          <div class="w-full text-center text-slate-500 py-16 select-none">
-            <svg class="w-12 h-12 mx-auto mb-3 opacity-40" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
-            <p>Enter text and click <span class="text-blue-400 font-medium">Tokenize</span> to see the breakdown.</p>
-            <p class="text-xs mt-1 opacity-60">Supports BPE, ByteLevel, WordPiece, Unigram (via transformers.js)</p>
-          </div>
-        </div>
-      </div>
     </div>
-    <footer class="mt-8 text-center text-slate-500 text-xs flex flex-col md:flex-row items-center justify-center gap-2">
-      <span>Powered by <code class="bg-slate-800 px-1.5 py-0.5 rounded text-blue-300">@huggingface/transformers</code> v3</span>
-      <span>•</span>
-      <span>Runs entirely client-side via WebAssembly/JS</span>
-      <span>•</span>
-      <span>No server/GPU required</span>
-    </footer>
-  </div>
-  <script type="module">
-    import { AutoTokenizer } from 'https://esm.sh/@huggingface/transformers@3.0.2';
-    const COLORS = [
-      '#3b82f6', '#8b5cf6', '#ec4899', '#f59e0b', '#10b981',
-      '#06b6d4', '#6366f1', '#f97316', '#84cc16', '#14b8a6'
-    ];
-    const els = {
-      modelId: document.getElementById('modelId'),
-      inputText: document.getElementById('inputText'),
-      btn: document.getElementById('tokenizeBtn'),
-      output: document.getElementById('outputGrid'),
-      status: document.getElementById('status'),
-      statTokens: document.getElementById('statTokens'),
-      statChars: document.getElementById('statChars'),
-      statRatio: document.getElementById('statRatio')
-    };
-    let tokenizerCache = null;
-    function getColor(idx) { return COLORS[idx % COLORS.length]; }
-    function setStatus(msg, type = 'info') {
-      els.status.textContent = msg;
-      els.status.className = `text-sm h-5 flex items-center gap-2 truncate ${type === 'error' ? 'text-red-400' : type === 'success' ? 'text-green-400' : 'text-slate-400'}`;
-    }
-    els.btn.addEventListener('click', async () => {
-      const modelId = els.modelId.value.trim();
-      const text = els.inputText.value;
-      if (!modelId) return setStatus('Please enter a Model ID.', 'error');
-      if (!text) return setStatus('Please enter text to tokenize.', 'error');
-      els.btn.disabled = true;
-      els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Loading...</span>';
-      els.output.innerHTML = '';
-      setStatus('Fetching tokenizer configuration...');
-      try {
-        if (!tokenizerCache || tokenizerCache.model_id !== modelId) {
-          setStatus('Downloading tokenizer (cached for session)...');
-          tokenizerCache = await AutoTokenizer.from_pretrained(modelId);
-          tokenizerCache.model_id = modelId;
-          setStatus('Tokenizer loaded successfully.', 'success');
         }
-        els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Tokenizing...</span>';
-        // Use backend tokenizer for precise offset mapping
-        const backend = tokenizerCache.backend_tokenizer || tokenizerCache.tokenizer;
-        const result = backend.encode(text);
-        renderTokens(result.tokens, result.ids, result.offsets, text);
-        setStatus('Done.', 'success');
-      } catch (err) {
-        console.error(err);
-        setStatus(`Error: ${err.message || 'Failed to load tokenizer.'}`, 'error');
-        els.output.innerHTML = `
-          <div class="w-full text-center py-12">
-            <div class="text-red-400 font-medium mb-2">Failed to Initialize</div>
-            <div class="text-slate-500 text-sm max-w-md mx-auto">
-              Ensure the Model ID is correct and the repository contains <code>tokenizer.json</code>.
-              Some gated models require a HF Token. Try <code>gpt2</code> as a fallback.
-            </div>
-          </div>`;
-      } finally {
-        els.btn.disabled = false;
-        els.btn.innerHTML = '<span>Tokenize</span>';
-      }
-    });
-    function renderTokens(tokens, ids, offsets, originalText) {
-      els.output.innerHTML = '';
-      const count = ids.length;
-      els.statTokens.textContent = count;
-      els.statChars.textContent = originalText.length;
-      els.statRatio.textContent = originalText.length > 0 ? (count / originalText.length).toFixed(2) : '0.00';
-      tokens.forEach((tok, i) => {
-        const [start, end] = offsets[i];
-        const chunk = originalText.slice(start, end);
-        const color = getColor(i);
-        const id = ids[i];
-        const chip = document.createElement('div');
-        chip.className = 'token-chip rounded-md px-2 py-1 text-sm font-mono border select-none fade-in';
-        chip.style.animationDelay = `${i * 0.015}s`;
-        chip.style.backgroundColor = `${color}15`;
-        chip.style.borderColor = `${color}80`;
-        chip.style.color = color;
-        chip.dataset.tooltip = `ID: ${id} | [${start}, ${end}) | "${chunk.replace(/"/g, '\\"')}"`;
-        // Handle invisible characters (like spaces, newlines) visually
-        const displayText = chunk === '' ? '␣' : chunk === '\n' ? '↵' : chunk;
-        chip.innerHTML = `<span class="font-medium">${displayText}</span><span class="opacity-50 text-[10px] ml-1 align-top">${id}</span>`;
-        els.output.appendChild(chip);
-      });
-    }
-    // Quick demo on load
-    window.addEventListener('load', () => {
-      if (!els.inputText.value) {
-        els.inputText.value = "Tokenization converts text into numerical IDs. DeepSeek-V4 uses advanced BPE!";
-      }
-    });
-  </script>
-</body>
-</html>

 <!DOCTYPE html>
+<html lang="en" class="light">
 <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>LLM Tokenizer Visualizer</title>
+    <!-- Tailwind CSS -->
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script>
+        tailwind.config = {
+            darkMode: 'class',
+            theme: {
+                extend: {
+                    fontFamily: {
+                        sans: ['Inter', 'sans-serif'],
+                        mono: ['Fira Code', 'monospace'],
+                    },
+                }
+            }
+        }
+    </script>
+    <!-- Google Fonts -->
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=Fira+Code:wght@400;500&display=swap" rel="stylesheet">
+    <!-- Transformers.js -->
+    <script type="module">
+        import { AutoTokenizer } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.1';
+        window.AutoTokenizer = AutoTokenizer;
+    </script>
+    <style type="text/tailwindcss">
+        @layer utilities {
+            .content-auto {
+                content-visibility: auto;
+            }
+            .token-box {
+                white-space: pre;
+                word-break: break-all;
+            }
+        }
+    </style>
 </head>
+<body class="bg-gray-50 dark:bg-gray-900 text-gray-900 dark:text-gray-100 min-h-screen transition-colors duration-200">
+    <div class="max-w-5xl mx-auto px-4 py-8">
+        <!-- Header -->
+        <header class="flex justify-between items-center mb-8">
+            <div>
+                <h1 class="text-3xl font-bold bg-gradient-to-r from-blue-600 to-purple-600 bg-clip-text text-transparent">LLM Tokenizer Visualizer</h1>
+                <p class="text-gray-600 dark:text-gray-400 mt-1">Client-side tokenization for any Hugging Face model, powered by Transformers.js</p>
+            </div>
+            <button id="themeToggle" class="p-2 rounded-full hover:bg-gray-200 dark:hover:bg-gray-800 transition-colors">
+                <!-- Sun Icon (for dark mode) -->
+                <svg id="sunIcon" class="w-6 h-6 hidden" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 3v1m0 16v1m9-9h-1M4 12H3m15.364 6.364l-.707-.707M6.343 6.343l-.707-.707m12.728 0l-.707.707M6.343 17.657l-.707.707M16 12a4 4 0 11-8 0 4 4 0 018 0z"></path>
+                </svg>
+                <!-- Moon Icon (for light mode) -->
+                <svg id="moonIcon" class="w-6 h-6" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M20.354 15.354A9 9 0 018.646 3.646 9.003 9.003 0 0012 21a9.003 9.003 0 008.354-5.646z"></path>
+                </svg>
+            </button>
+        </header>
+        <!-- Model Config Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <h2 class="text-lg font-semibold mb-4">Model Configuration</h2>
+            <div class="grid grid-cols-1 md:grid-cols-12 gap-4">
+                <div class="md:col-span-8">
+                    <label class="block text-sm font-medium mb-2" for="modelId">Hugging Face Model ID</label>
+                    <div class="flex gap-2">
+                        <input
+                            type="text"
+                            id="modelId"
+                            value="deepseek-ai/DeepSeek-V4-Pro"
+                            class="flex-1 px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                            placeholder="e.g. meta-llama/Meta-Llama-3-8B-Instruct"
+                        >
+                        <button
+                            id="loadModelBtn"
+                            class="px-6 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors flex items-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed"
+                        >
+                            <svg id="loadSpinner" class="animate-spin w-4 h-4 hidden" fill="none" viewBox="0 0 24 24">
+                                <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
+                                <path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
+                            </svg>
+                            <span id="loadBtnText">Load Model</span>
+                        </button>
+                    </div>
+                </div>
+                <div class="md:col-span-4">
+                    <label class="block text-sm font-medium mb-2" for="hfToken">Optional HF Token (for gated models)</label>
+                    <input
+                        type="password"
+                        id="hfToken"
+                        class="w-full px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                        placeholder="hf_..."
+                    >
+                </div>
+            </div>
+            <div class="mt-4">
+                <label class="block text-sm font-medium mb-2">Popular Models</label>
+                <select
+                    id="popularModels"
+                    class="w-full md:w-1/3 px-4 py-2 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all"
+                >
+                    <option value="">Select a popular model...</option>
+                    <option value="deepseek-ai/DeepSeek-V4-Pro">DeepSeek-V4-Pro</option>
+                    <option value="meta-llama/Meta-Llama-3-8B-Instruct">Llama 3 8B Instruct</option>
+                    <option value="mistralai/Mistral-7B-Instruct-v0.3">Mistral 7B v0.3</option>
+                    <option value="google/gemma-2-2b-it">Gemma 2 2B IT</option>
+                    <option value="Qwen/Qwen2.5-7B-Instruct">Qwen 2.5 7B Instruct</option>
+                    <option value="microsoft/Phi-3-mini-4k-instruct">Phi-3 Mini 4K Instruct</option>
+                </select>
+            </div>
+            <div id="modelError" class="hidden mt-3 p-3 bg-red-100 dark:bg-red-900/30 text-red-700 dark:text-red-300 rounded-lg text-sm"></div>
+            <div id="modelSuccess" class="hidden mt-3 p-3 bg-green-100 dark:bg-green-900/30 text-green-700 dark:text-green-300 rounded-lg text-sm"></div>
+        </section>
+        <!-- Text Input Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <div class="flex justify-between items-center mb-4">
+                <h2 class="text-lg font-semibold">Input Text</h2>
+                <button
+                    id="clearTextBtn"
+                    class="text-sm text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100 transition-colors"
+                >
+                    Clear
+                </button>
             </div>
+            <textarea
+                id="textInput"
+                rows="4"
+                class="w-full px-4 py-3 rounded-lg border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-700 focus:ring-2 focus:ring-blue-500 focus:border-transparent outline-none transition-all font-mono text-sm resize-none"
+                placeholder="Enter text to tokenize..."
+                disabled
+            >Hello, world! This is a test of the tokenizer. Let's see how many tokens this takes.
+你好，世界！这是一个分词器测试。
+Special tokens: <|endoftext|> <s> </s></textarea>
+        </section>
+        <!-- Stats Section -->
+        <section class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Total Tokens</p>
+                <p id="totalTokens" class="text-2xl font-bold mt-1">0</p>
+            </div>
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Total Characters</p>
+                <p id="totalChars" class="text-2xl font-bold mt-1">0</p>
             </div>
+            <div class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-4">
+                <p class="text-sm text-gray-600 dark:text-gray-400">Tokens per Character</p>
+                <p id="tokenRatio" class="text-2xl font-bold mt-1">0.00</p>
+            </div>
+        </section>
+        <!-- Token Visualization Section -->
+        <section class="bg-white dark:bg-gray-800 rounded-xl shadow-sm p-6 mb-6">
+            <div class="flex justify-between items-center mb-4">
+                <h2 class="text-lg font-semibold">Token Visualization</h2>
+                <div class="flex gap-2">
+                    <button
+                        id="copyTokensBtn"
+                        class="px-4 py-2 bg-gray-100 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-600 rounded-lg text-sm font-medium transition-colors flex items-center gap-2 disabled:opacity-50"
+                        disabled
+                    >
+                        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
+                        </svg>
+                        Copy Tokens
+                    </button>
+                    <button
+                        id="copyIdsBtn"
+                        class="px-4 py-2 bg-gray-100 dark:bg-gray-700 hover:bg-gray-200 dark:hover:bg-gray-600 rounded-lg text-sm font-medium transition-colors flex items-center gap-2 disabled:opacity-50"
+                        disabled
+                    >
+                        <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z"></path>
+                        </svg>
+                        Copy IDs
+                    </button>
+                </div>
             </div>
+            <div
+                id="tokenContainer"
+                class="min-h-[120px] p-4 bg-gray-50 dark:bg-gray-700/50 rounded-lg overflow-x-auto flex flex-wrap gap-2 items-start content-start"
+            >
+                <p id="emptyState" class="text-gray-500 dark:text-gray-400 w-full text-center py-8">Load a model and enter text to see tokens</p>
+            </div>
+            <p class="text-xs text-gray-500 dark:text-gray-400 mt-2">Hover over tokens to see their ID. Special tokens are marked with a dashed border.</p>
+        </section>
+        <!-- Footer -->
+        <footer class="text-center text-gray-600 dark:text-gray-400 text-sm">
+            <p>Powered by <a href="https://huggingface.co/docs/transformers.js" target="_blank" class="text-blue-600 dark:text-blue-400 hover:underline">Transformers.js</a> | Runs entirely in your browser, no server required</p>
+        </footer>
     </div>
+    <script type="module">
+        // Wait for Transformers.js to load
+        await new Promise(resolve => {
+            const checkInterval = setInterval(() => {
+                if (window.AutoTokenizer) clearInterval(checkInterval);
+            }, 100);
+        });
+        const { AutoTokenizer } = window;
+        // DOM Elements
+        const themeToggle = document.getElementById('themeToggle');
+        const sunIcon = document.getElementById('sunIcon');
+        const moonIcon = document.getElementById('moonIcon');
+        const modelIdInput = document.getElementById('modelId');
+        const hfTokenInput = document.getElementById('hfToken');
+        const loadModelBtn = document.getElementById('loadModelBtn');
+        const loadSpinner = document.getElementById('loadSpinner');
+        const loadBtnText = document.getElementById('loadBtnText');
+        const popularModelsSelect = document.getElementById('popularModels');
+        const modelError = document.getElementById('modelError');
+        const modelSuccess = document.getElementById('modelSuccess');
+        const textInput = document.getElementById('textInput');
+        const clearTextBtn = document.getElementById('clearTextBtn');
+        const tokenContainer = document.getElementById('tokenContainer');
+        const emptyState = document.getElementById('emptyState');
+        const totalTokensEl = document.getElementById('totalTokens');
+        const totalCharsEl = document.getElementById('totalChars');
+        const tokenRatioEl = document.getElementById('tokenRatio');
+        const copyTokensBtn = document.getElementById('copyTokensBtn');
+        const copyIdsBtn = document.getElementById('copyIdsBtn');
+        // State
+        let tokenizer = null;
+        let currentTokens = [];
+        let currentTokenIds = [];
+        let isDark = localStorage.getItem('theme') === 'dark' || (!localStorage.getItem('theme') && window.matchMedia('(prefers-color-scheme: dark)').matches);
+        // Initialize theme
+        function initTheme() {
+            if (isDark) {
+                document.documentElement.classList.add('dark');
+                sunIcon.classList.remove('hidden');
+                moonIcon.classList.add('hidden');
+            } else {
+                document.documentElement.classList.remove('dark');
+                sunIcon.classList.add('hidden');
+                moonIcon.classList.remove('hidden');
+            }
         }
+        initTheme();
+        // Theme toggle
+        themeToggle.addEventListener('click', () => {
+            isDark = !isDark;
+            localStorage.setItem('theme', isDark ? 'dark' : 'light');
+            initTheme();
+            // Re-render tokens to update colors
+            if (currentTokens.length > 0) renderTokens();
+        });
+        // Popular models select
+        popularModelsSelect.addEventListener('change', (e) => {
+            if (e.target.value) {
+                modelIdInput.value = e.target.value;
+            }
+        });
+        // Load model function
+        async function loadModel() {
+            const modelId = modelIdInput.value.trim();
+            if (!modelId) {
+                showError('Please enter a Hugging Face model ID');
+                return;
+            }
+            // Show loading state
+            loadModelBtn.disabled = true;
+            loadSpinner.classList.remove('hidden');
+            loadBtnText.textContent = 'Loading...';
+            hideMessages();
+            textInput.disabled = true;
+            try {
+                const token = hfTokenInput.value.trim() || undefined;
+                tokenizer = await AutoTokenizer.from_pretrained(modelId, { token });
+                // Success
+                showSuccess(`Successfully loaded tokenizer for ${modelId}`);
+                textInput.disabled = false;
+                textInput.focus();
+                // Auto-tokenize existing text
+                if (textInput.value.trim()) {
+                    tokenizeText();
+                }
+            } catch (err) {
+                console.error('Failed to load model:', err);
+                showError(`Failed to load model: ${err.message}. Check the model ID and ensure it's public (or provide a HF token for gated models).`);
+                tokenizer = null;
+                textInput.disabled = true;
+            } finally {
+                // Reset button state
+                loadModelBtn.disabled = false;
+                loadSpinner.classList.add('hidden');
+                loadBtnText.textContent = 'Load Model';
+            }
+        }
+        // Tokenize text function
+        function tokenizeText() {
+            if (!tokenizer) return;
+            const text = textInput.value;
+            if (!text.trim()) {
+                currentTokens = [];
+                currentTokenIds = [];
+                renderTokens();
+                updateStats();
+                copyTokensBtn.disabled = true;
+                copyIdsBtn.disabled = true;
+                return;
+            }
+            try {
+                const output = tokenizer(text);
+                currentTokens = output.tokens;
+                currentTokenIds = output.input_ids;
+                renderTokens();
+                updateStats();
+                copyTokensBtn.disabled = false;
+                copyIdsBtn.disabled = false;
+            } catch (err) {
+                console.error('Tokenization failed:', err);
+                showError(`Tokenization failed: ${err.message}`);
+            }
+        }
+        // Render tokens
+        function renderTokens() {
+            tokenContainer.innerHTML = '';
+            if (currentTokens.length === 0) {
+                tokenContainer.appendChild(emptyState);
+                emptyState.classList.remove('hidden');
+                return;
+            }
+            emptyState.classList.add('hidden');
+            currentTokens.forEach((token, index) => {
+                const tokenId = currentTokenIds[index];
+                // Generate consistent pastel color based on token ID
+                const hue = (tokenId * 137) % 360; // 137 is prime for even distribution
+                const lightness = isDark ? '80%' : '90%';
+                const bgColor = `hsl(${hue}, 70%, ${lightness})`;
+                const textColor = isDark ? '#1f2937' : '#1f2937';
+                const tokenEl = document.createElement('span');
+                tokenEl.className = 'token-box px-2 py-1 rounded-md text-sm font-mono cursor-pointer transition-all hover:scale-105 hover:shadow-md';
+                tokenEl.style.backgroundColor = bgColor;
+                tokenEl.style.color = textColor;
+                // Replace whitespace with visible symbols
+                const displayToken = token
+                    .replace(/\n/g, '↵')
+                    .replace(/\t/g, '→')
+                    .replace(/ /g, '·');
+                tokenEl.textContent = displayToken;
+                // Show token ID on hover
+                tokenEl.title = `Token ID: ${tokenId}\nRaw: ${token.replace(/\n/g, '\\n').replace(/\t/g, '\\t')}`;
+                // Highlight special tokens
+                if (tokenizer.special_tokens_map && Object.values(tokenizer.special_tokens_map).flat().includes(token)) {
+                    tokenEl.classList.add('border', 'border-dashed', 'border-gray-400', 'dark:border-gray-500');
+                }
+                tokenContainer.appendChild(tokenEl);
+            });
+        }
+        // Update stats
+        function updateStats() {
+            const totalTokens = currentTokens.length;
+            const totalChars = textInput.value.length;
+            const ratio = totalChars > 0 ? (totalTokens / totalChars).toFixed(2) : '0.00';
+            totalTokensEl.textContent = totalTokens.toLocaleString();
+            totalCharsEl.textContent = totalChars.toLocaleString();
+            tokenRatioEl.textContent = ratio;
+        }
+        // Copy functions
+        async function copyToClipboard(text, btn) {
+            try {
+                await navigator.clipboard.writeText(text);
+                const originalText = btn.innerHTML;
+                btn.innerHTML = `
+                    <svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+                        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M5 13l4 4L19 7"></path>
+                    </svg>
+                    Copied!
+                `;
+                setTimeout(() => {
+                    btn.innerHTML = originalText;
+                }, 2000);
+            } catch (err) {
+                showError('Failed to copy to clipboard');
+            }
+        }
+        copyTokensBtn.addEventListener('click', () => {
+            copyToClipboard(currentTokens.join('\n'), copyTokensBtn);
+        });
+        copyIdsBtn.addEventListener('click', () => {
+            copyToClipboard(JSON.stringify(currentTokenIds, null, 2), copyIdsBtn);
+        });
+        // Clear text
+        clearTextBtn.addEventListener('click', () => {
+            textInput.value = '';
+            tokenizeText();
+        });
+        // Helper functions
+        function showError(message) {
+            modelError.textContent = message;
+            modelError.classList.remove('hidden');
+            modelSuccess.classList.add('hidden');
+        }
+        function showSuccess(message) {
+            modelSuccess.textContent = message;
+            modelSuccess.classList.remove('hidden');
+            modelError.classList.add('hidden');
+        }
+        function hideMessages() {
+            modelError.classList.add('hidden');
+            modelSuccess.classList.add('hidden');
+        }
+        // Debounce for text input
+        function debounce(func, wait) {
+            let timeout;
+            return function(...args) {
+                clearTimeout(timeout);
+                timeout = setTimeout(() => func.apply(this, args), wait);
+            };
+        }
+        const debouncedTokenize = debounce(tokenizeText, 300);
+        // Event listeners
+        loadModelBtn.addEventListener('click', loadModel);
+        modelIdInput.addEventListener('keypress', (e) => {
+            if (e.key === 'Enter') loadModel();
+        });
+        textInput.addEventListener('input', debouncedTokenize);
+        // Auto-load default model on page load
+        window.addEventListener('load', () => {
+            // Wait a bit for Transformers.js to fully initialize
+            setTimeout(loadModel, 500);
+        });
+    </script>
+</body>
+</html>