Spaces:

quickgrid
/

Tokenizer-Visualizer

Running

App Files Files Community

quickgrid commited on 5 days ago

Commit

fb8c813

verified ·

1 Parent(s): 36774ea

Update index.html

Browse files

Files changed (1) hide show

index.html +248 -18

index.html CHANGED Viewed

@@ -1,19 +1,249 @@
-<!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Universal Tokenizer Visualizer</title>
+  <script src="https://cdn.tailwindcss.com"></script>
+  <link href="https://cdn.jsdelivr.net/npm/@fontsource/inter@5.0.16/index.min.css" rel="stylesheet">
+  <style>
+    body { font-family: 'Inter', sans-serif; background-color: #0f172a; }
+    .token-chip {
+      transition: all 0.15s cubic-bezier(0.4, 0, 0.2, 1);
+      cursor: default;
+      position: relative;
+    }
+    .token-chip:hover {
+      transform: translateY(-2px) scale(1.03);
+      z-index: 20;
+      box-shadow: 0 8px 16px -4px rgba(0,0,0,0.5);
+    }
+    .token-chip::after {
+      content: attr(data-tooltip);
+      position: absolute;
+      bottom: 110%;
+      left: 50%;
+      transform: translateX(-50%) translateY(5px);
+      background: #1e293b;
+      color: #e2e8f0;
+      padding: 6px 10px;
+      border-radius: 6px;
+      font-size: 0.75rem;
+      white-space: nowrap;
+      opacity: 0;
+      pointer-events: none;
+      transition: all 0.2s;
+      border: 1px solid #334155;
+      box-shadow: 0 4px 12px rgba(0,0,0,0.3);
+      z-index: 30;
+    }
+    .token-chip:hover::after { opacity: 1; transform: translateX(-50%) translateY(0); }
+    .spinner {
+      width: 20px; height: 20px;
+      border: 2px solid rgba(255,255,255,0.3);
+      border-radius: 50%;
+      border-top-color: #fff;
+      animation: spin 0.8s linear infinite;
+    }
+    @keyframes spin { to { transform: rotate(360deg); } }
+    ::-webkit-scrollbar { width: 8px; height: 8px; }
+    ::-webkit-scrollbar-track { background: #1e293b; border-radius: 4px; }
+    ::-webkit-scrollbar-thumb { background: #475569; border-radius: 4px; }
+    ::-webkit-scrollbar-thumb:hover { background: #64748b; }
+    .fade-in { animation: fadeIn 0.3s ease-out forwards; }
+    @keyframes fadeIn { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } }
+  </style>
+</head>
+<body class="min-h-screen text-slate-100 flex flex-col">
+  <div class="flex-grow max-w-6xl w-full mx-auto p-4 md:p-8">
+    <header class="mb-8 flex flex-col md:flex-row md:items-end justify-between gap-4">
+      <div>
+        <h1 class="text-3xl md:text-4xl font-bold bg-gradient-to-r from-cyan-400 via-blue-500 to-purple-500 bg-clip-text text-transparent tracking-tight">
+          🧩 Tokenizer Visualizer
+        </h1>
+        <p class="text-slate-400 mt-2 text-sm md:text-base">
+          Load any HuggingFace model tokenizer directly in your browser. Client-side only, no GPU required.
+        </p>
+      </div>
+      <div class="flex gap-2 text-xs text-slate-500">
+        <span class="px-2 py-1 bg-slate-800 rounded border border-slate-700">Browser-Native</span>
+        <span class="px-2 py-1 bg-slate-800 rounded border border-slate-700">Extensible</span>
+      </div>
+    </header>
+    <div class="grid grid-cols-1 lg:grid-cols-12 gap-6">
+      <!-- Controls Panel -->
+      <div class="lg:col-span-4 bg-slate-800/50 backdrop-blur border border-slate-700 rounded-xl p-5 flex flex-col gap-4 h-fit shadow-xl">
+        <div>
+          <label class="block text-sm font-medium text-slate-300 mb-1.5">Model ID / Path</label>
+          <input type="text" id="modelId" value="gpt2"
+            class="w-full bg-slate-900 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 transition"
+            placeholder="e.g., gpt2, deepseek-ai/DeepSeek-V3, meta-llama/Llama-3-8B">
+        </div>
+        <div class="flex-grow">
+          <label class="block text-sm font-medium text-slate-300 mb-1.5">Input Text</label>
+          <textarea id="inputText" rows="7"
+            class="w-full bg-slate-900 border border-slate-600 rounded-lg px-3 py-2.5 text-sm focus:outline-none focus:ring-2 focus:ring-blue-500 resize-y transition"
+            placeholder="Type or paste text to visualize tokenization..."></textarea>
+        </div>
+        <button id="tokenizeBtn"
+          class="w-full bg-blue-600 hover:bg-blue-500 active:bg-blue-700 text-white font-semibold py-2.5 px-4 rounded-lg transition flex items-center justify-center gap-2 disabled:opacity-50 disabled:cursor-not-allowed shadow-lg shadow-blue-900/30">
+          <span>Tokenize</span>
+        </button>
+        <div id="status" class="text-sm text-slate-400 h-5 flex items-center gap-2 truncate"></div>
+      </div>
+      <!-- Visualization Panel -->
+      <div class="lg:col-span-8 bg-slate-800/50 backdrop-blur border border-slate-700 rounded-xl p-5 flex flex-col shadow-xl min-h-[450px]">
+        <div class="flex flex-wrap justify-between items-center mb-4 pb-3 border-b border-slate-700 gap-3">
+          <h2 class="text-lg font-semibold text-slate-100">Token Breakdown</h2>
+          <div class="flex flex-wrap gap-3 text-sm">
+            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
+              <span class="w-2 h-2 rounded-full bg-blue-400"></span>
+              <span class="text-slate-300">Tokens: <b id="statTokens" class="text-white">0</b></span>
+            </div>
+            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
+              <span class="w-2 h-2 rounded-full bg-purple-400"></span>
+              <span class="text-slate-300">Chars: <b id="statChars" class="text-white">0</b></span>
+            </div>
+            <div class="flex items-center gap-1.5 bg-slate-900 px-3 py-1.5 rounded-md border border-slate-700">
+              <span class="w-2 h-2 rounded-full bg-pink-400"></span>
+              <span class="text-slate-300">Ratio: <b id="statRatio" class="text-white">0.00</b></span>
+            </div>
+          </div>
+        </div>
+        <div id="outputGrid" class="flex flex-wrap gap-2 overflow-y-auto p-2 content-start flex-grow min-h-[300px] bg-slate-900/40 rounded-lg border border-slate-800/50">
+          <div class="w-full text-center text-slate-500 py-16 select-none">
+            <svg class="w-12 h-12 mx-auto mb-3 opacity-40" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M13 10V3L4 14h7v7l9-11h-7z"></path></svg>
+            <p>Enter text and click <span class="text-blue-400 font-medium">Tokenize</span> to see the breakdown.</p>
+            <p class="text-xs mt-1 opacity-60">Supports BPE, ByteLevel, WordPiece, Unigram (via transformers.js)</p>
+          </div>
+        </div>
+      </div>
+    </div>
+    <footer class="mt-8 text-center text-slate-500 text-xs flex flex-col md:flex-row items-center justify-center gap-2">
+      <span>Powered by <code class="bg-slate-800 px-1.5 py-0.5 rounded text-blue-300">@huggingface/transformers</code> v3</span>
+      <span>•</span>
+      <span>Runs entirely client-side via WebAssembly/JS</span>
+      <span>•</span>
+      <span>No server/GPU required</span>
+    </footer>
+  </div>
+  <script type="module">
+    import { AutoTokenizer } from 'https://esm.sh/@huggingface/transformers@3.0.2';
+    const COLORS = [
+      '#3b82f6', '#8b5cf6', '#ec4899', '#f59e0b', '#10b981',
+      '#06b6d4', '#6366f1', '#f97316', '#84cc16', '#14b8a6'
+    ];
+    const els = {
+      modelId: document.getElementById('modelId'),
+      inputText: document.getElementById('inputText'),
+      btn: document.getElementById('tokenizeBtn'),
+      output: document.getElementById('outputGrid'),
+      status: document.getElementById('status'),
+      statTokens: document.getElementById('statTokens'),
+      statChars: document.getElementById('statChars'),
+      statRatio: document.getElementById('statRatio')
+    };
+    let tokenizerCache = null;
+    function getColor(idx) { return COLORS[idx % COLORS.length]; }
+    function setStatus(msg, type = 'info') {
+      els.status.textContent = msg;
+      els.status.className = `text-sm h-5 flex items-center gap-2 truncate ${type === 'error' ? 'text-red-400' : type === 'success' ? 'text-green-400' : 'text-slate-400'}`;
+    }
+    els.btn.addEventListener('click', async () => {
+      const modelId = els.modelId.value.trim();
+      const text = els.inputText.value;
+      if (!modelId) return setStatus('Please enter a Model ID.', 'error');
+      if (!text) return setStatus('Please enter text to tokenize.', 'error');
+      els.btn.disabled = true;
+      els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Loading...</span>';
+      els.output.innerHTML = '';
+      setStatus('Fetching tokenizer configuration...');
+      try {
+        if (!tokenizerCache || tokenizerCache.model_id !== modelId) {
+          setStatus('Downloading tokenizer (cached for session)...');
+          tokenizerCache = await AutoTokenizer.from_pretrained(modelId);
+          tokenizerCache.model_id = modelId;
+          setStatus('Tokenizer loaded successfully.', 'success');
+        }
+        els.btn.innerHTML = '<div class="spinner"></div> <span class="ml-2">Tokenizing...</span>';
+        // Use backend tokenizer for precise offset mapping
+        const backend = tokenizerCache.backend_tokenizer || tokenizerCache.tokenizer;
+        const result = backend.encode(text);
+        renderTokens(result.tokens, result.ids, result.offsets, text);
+        setStatus('Done.', 'success');
+      } catch (err) {
+        console.error(err);
+        setStatus(`Error: ${err.message || 'Failed to load tokenizer.'}`, 'error');
+        els.output.innerHTML = `
+          <div class="w-full text-center py-12">
+            <div class="text-red-400 font-medium mb-2">Failed to Initialize</div>
+            <div class="text-slate-500 text-sm max-w-md mx-auto">
+              Ensure the Model ID is correct and the repository contains <code>tokenizer.json</code>.
+              Some gated models require a HF Token. Try <code>gpt2</code> as a fallback.
+            </div>
+          </div>`;
+      } finally {
+        els.btn.disabled = false;
+        els.btn.innerHTML = '<span>Tokenize</span>';
+      }
+    });
+    function renderTokens(tokens, ids, offsets, originalText) {
+      els.output.innerHTML = '';
+      const count = ids.length;
+      els.statTokens.textContent = count;
+      els.statChars.textContent = originalText.length;
+      els.statRatio.textContent = originalText.length > 0 ? (count / originalText.length).toFixed(2) : '0.00';
+      tokens.forEach((tok, i) => {
+        const [start, end] = offsets[i];
+        const chunk = originalText.slice(start, end);
+        const color = getColor(i);
+        const id = ids[i];
+        const chip = document.createElement('div');
+        chip.className = 'token-chip rounded-md px-2 py-1 text-sm font-mono border select-none fade-in';
+        chip.style.animationDelay = `${i * 0.015}s`;
+        chip.style.backgroundColor = `${color}15`;
+        chip.style.borderColor = `${color}80`;
+        chip.style.color = color;
+        chip.dataset.tooltip = `ID: ${id} | [${start}, ${end}) | "${chunk.replace(/"/g, '\\"')}"`;
+        // Handle invisible characters (like spaces, newlines) visually
+        const displayText = chunk === '' ? '␣' : chunk === '\n' ? '↵' : chunk;
+        chip.innerHTML = `<span class="font-medium">${displayText}</span><span class="opacity-50 text-[10px] ml-1 align-top">${id}</span>`;
+        els.output.appendChild(chip);
+      });
+    }
+    // Quick demo on load
+    window.addEventListener('load', () => {
+      if (!els.inputText.value) {
+        els.inputText.value = "Tokenization converts text into numerical IDs. DeepSeek-V4 uses advanced BPE!";
+      }
+    });
+  </script>
+</body>
 </html>