Spaces:
Running
Running
| <html lang="en" data-theme="dark"> | |
| <head> | |
| <meta charset="UTF-8" /> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0" /> | |
| <title>TokenLens β LLM Tokenizer Playground</title> | |
| <meta name="description" content="Visualize how large language models tokenize text. Powered by Transformers.js, runs entirely in your browser." /> | |
| <link rel="preconnect" href="https://fonts.googleapis.com" /> | |
| <link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,300;12..96,400;12..96,500;12..96,600;12..96,700;12..96,800&family=JetBrains+Mono:wght@300;400;500;700&family=DM+Sans:wght@300;400;500&display=swap" rel="stylesheet" /> | |
| <style> | |
| /* βββ Design Tokens βββββββββββββββββββββββββββββββββββ */ | |
| :root { | |
| --bg: #060b14; | |
| --bg2: #0b1220; | |
| --bg3: #101828; | |
| --bg4: #162035; | |
| --border: #1a2d4a; | |
| --border2: #243d60; | |
| --glow: #1f3d6e; | |
| --text: #dce8f8; | |
| --text2: #7899c0; | |
| --text3: #3d5a80; | |
| --accent: #4d9ef5; | |
| --accent2: #8b6af5; | |
| --green: #34d89a; | |
| --amber: #f5a623; | |
| --red: #f55577; | |
| } | |
| [data-theme="light"] { | |
| --bg: #eef1f8; | |
| --bg2: #e2e7f2; | |
| --bg3: #d3dae8; | |
| --bg4: #c2cce0; | |
| --border: #b8c4d8; | |
| --border2: #a0aec8; | |
| --glow: #c0d0e8; | |
| --text: #1a2236; | |
| --text2: #5a6888; | |
| --text3: #8898b4; | |
| --accent: #2878e0; | |
| --accent2: #6838d8; | |
| --green: #18a060; | |
| --amber: #c88010; | |
| --red: #d83858; | |
| } | |
| /* βββ Reset βββββββββββββββββββββββββββββββββββββββββββ */ | |
| *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } | |
| html { height: 100%; overflow: hidden; } | |
| body { | |
| background: var(--bg); | |
| color: var(--text); | |
| font-family: 'DM Sans', sans-serif; | |
| height: 100%; | |
| overflow: hidden; | |
| } | |
| /* βββ Background FX βββββββββββββββββββββββββββββββββββ */ | |
| .bg-gradient { | |
| position: fixed; inset: 0; pointer-events: none; z-index: 0; | |
| background: | |
| radial-gradient(ellipse 80% 50% at 20% 10%, rgba(77,158,245,.06) 0%, transparent 70%), | |
| radial-gradient(ellipse 60% 40% at 80% 90%, rgba(139,106,245,.05) 0%, transparent 60%), | |
| radial-gradient(ellipse 40% 30% at 60% 50%, rgba(52,216,154,.03) 0%, transparent 60%); | |
| } | |
| [data-theme="light"] .bg-gradient { | |
| background: | |
| radial-gradient(ellipse 80% 50% at 20% 10%, rgba(40,120,224,.05) 0%, transparent 70%), | |
| radial-gradient(ellipse 60% 40% at 80% 90%, rgba(104,56,216,.04) 0%, transparent 60%), | |
| radial-gradient(ellipse 40% 30% at 60% 50%, rgba(24,160,96,.02) 0%, transparent 60%); | |
| } | |
| .dot-grid { | |
| position: fixed; inset: 0; pointer-events: none; z-index: 0; | |
| background-image: radial-gradient(circle, rgba(77,158,245,.12) 1px, transparent 1px); | |
| background-size: 36px 36px; | |
| mask-image: radial-gradient(ellipse 100% 100% at 50% 50%, black 30%, transparent 80%); | |
| } | |
| [data-theme="light"] .dot-grid { | |
| background-image: radial-gradient(circle, rgba(40,120,224,.06) 1px, transparent 1px); | |
| } | |
| /* βββ Layout βββββββββββββββββββββββββββββββββββββββββββ */ | |
| #app { | |
| position: relative; z-index: 1; | |
| display: flex; flex-direction: column; | |
| height: 100vh; overflow: hidden; | |
| } | |
| /* βββ Header βββββββββββββββββββββββββββββββββββββββββββ */ | |
| header { | |
| display: flex; align-items: center; | |
| padding: 0 20px; height: 56px; | |
| border-bottom: 1px solid var(--border); | |
| background: rgba(6,11,20,.85); | |
| backdrop-filter: blur(20px); | |
| flex-shrink: 0; z-index: 100; gap: 12px; | |
| } | |
| [data-theme="light"] header { background: rgba(238,241,248,.92); } | |
| .logo { | |
| display: flex; align-items: center; gap: 8px; | |
| text-decoration: none; color: var(--text); flex-shrink: 0; | |
| } | |
| .logo-hex { | |
| width: 30px; height: 30px; | |
| background: linear-gradient(135deg, var(--accent), var(--accent2)); | |
| clip-path: polygon(50% 0%, 93% 25%, 93% 75%, 50% 100%, 7% 75%, 7% 25%); | |
| display: flex; align-items: center; justify-content: center; | |
| font-size: 12px; font-family: 'JetBrains Mono', monospace; font-weight: 700; color: white; | |
| } | |
| .logo-name { | |
| font-family: 'Bricolage Grotesque', sans-serif; | |
| font-size: 17px; font-weight: 700; letter-spacing: -0.5px; | |
| background: linear-gradient(135deg, #dce8f8 40%, var(--accent)); | |
| -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; | |
| } | |
| [data-theme="light"] .logo-name { | |
| background: linear-gradient(135deg, #1a2236 40%, var(--accent)); | |
| -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; | |
| } | |
| .logo-tag { | |
| font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3); | |
| background: var(--bg3); border: 1px solid var(--border); | |
| padding: 1px 5px; border-radius: 4px; letter-spacing: .5px; | |
| } | |
| .header-divider { | |
| width: 1px; height: 28px; background: var(--border); flex-shrink: 0; | |
| } | |
| /* βββ Search Bar Groups βββββββββββββββββββββββββββββββ */ | |
| .header-controls { | |
| display: flex; align-items: center; gap: 10px; flex: 1; min-width: 0; | |
| } | |
| .searchbar-group { | |
| position: relative; display: flex; align-items: center; gap: 3px; flex: 1; min-width: 0; | |
| } | |
| .searchbar-label { | |
| font-family: 'JetBrains Mono', monospace; font-size: 11px; font-weight: 700; | |
| width: 22px; height: 22px; border-radius: 5px; display: flex; align-items: center; | |
| justify-content: center; flex-shrink: 0; border: 1px solid var(--border); | |
| } | |
| .searchbar-label.label-a { background: rgba(77,158,245,.15); color: var(--accent); border-color: rgba(77,158,245,.3); } | |
| .searchbar-label.label-b { background: rgba(139,106,245,.15); color: var(--accent2); border-color: rgba(139,106,245,.3); } | |
| .searchbar-input { | |
| flex: 1; min-width: 0; background: var(--bg2); border: 1px solid var(--border); | |
| border-radius: 6px; color: var(--text); font-family: 'JetBrains Mono', monospace; | |
| font-size: 11px; padding: 5px 8px; outline: none; transition: border-color .2s; | |
| } | |
| .searchbar-input:focus { border-color: var(--accent); } | |
| .searchbar-input::placeholder { color: var(--text3); } | |
| .searchbar-dropdown-btn { | |
| width: 24px; height: 24px; border-radius: 5px; border: 1px solid var(--border); | |
| background: var(--bg2); color: var(--text2); cursor: pointer; font-size: 10px; | |
| display: flex; align-items: center; justify-content: center; flex-shrink: 0; | |
| transition: all .15s; | |
| } | |
| .searchbar-dropdown-btn:hover { border-color: var(--border2); color: var(--text); } | |
| .searchbar-load-btn { | |
| padding: 4px 10px; border-radius: 5px; border: 1px solid var(--border2); | |
| background: linear-gradient(135deg, rgba(77,158,245,.12), rgba(139,106,245,.12)); | |
| color: var(--accent); font-family: 'DM Sans', sans-serif; font-size: 11px; | |
| font-weight: 500; cursor: pointer; transition: all .15s; white-space: nowrap; flex-shrink: 0; | |
| } | |
| .searchbar-load-btn:hover { | |
| background: linear-gradient(135deg, rgba(77,158,245,.22), rgba(139,106,245,.22)); | |
| border-color: var(--accent); | |
| } | |
| /* βββ Dropdown Menu βββββββββββββββββββββββββββββββββββ */ | |
| .dropdown-menu { | |
| position: absolute; top: calc(100% + 6px); left: 22px; right: 0; | |
| min-width: 280px; max-height: 320px; overflow-y: auto; | |
| background: var(--bg3); border: 1px solid var(--border2); border-radius: 8px; | |
| z-index: 200; display: none; padding: 4px; | |
| box-shadow: 0 8px 32px rgba(0,0,0,.4); | |
| } | |
| [data-theme="light"] .dropdown-menu { box-shadow: 0 8px 32px rgba(0,0,0,.12); } | |
| .dropdown-menu.open { display: block; } | |
| .dropdown-item { | |
| padding: 7px 10px; cursor: pointer; border-radius: 5px; | |
| display: flex; align-items: center; gap: 8px; font-size: 12px; | |
| transition: background .12s; | |
| } | |
| .dropdown-item:hover { background: var(--bg4); } | |
| .dropdown-item-dot { width: 6px; height: 6px; border-radius: 50%; flex-shrink: 0; } | |
| .dropdown-item-name { font-weight: 600; color: var(--text); font-family: 'Bricolage Grotesque', sans-serif; } | |
| .dropdown-item-detail { font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-left: auto; white-space: nowrap; } | |
| /* βββ Icon Toggle Buttons ββββββββββββββββββββββββββββββ */ | |
| .icon-toggle-btn { | |
| width: 24px; height: 24px; border-radius: 7px; border: 1px solid var(--border); | |
| background: var(--bg2); color: var(--text2); cursor: pointer; flex-shrink: 0; | |
| display: flex; align-items: center; justify-content: center; transition: all .15s; | |
| } | |
| .icon-toggle-btn:hover { border-color: var(--border2); color: var(--text); } | |
| .icon-toggle-btn.active { border-color: var(--accent); color: var(--accent); background: rgba(77,158,245,.1); } | |
| .icon-toggle-btn svg { width: 16px; height: 16px; } | |
| /* βββ Mobile Tab Bar βββββββββββββββββββββββββββββββββββ */ | |
| .mobile-tab-bar { | |
| display: none; | |
| border-bottom: 1px solid var(--border); | |
| flex-shrink: 0; | |
| background: var(--bg2); | |
| padding: 0 4px; | |
| } | |
| .mobile-tab { | |
| flex: 1; | |
| padding: 10px 4px; | |
| border: none; | |
| background: transparent; | |
| color: var(--text2); | |
| font-family: 'DM Sans', sans-serif; | |
| font-size: 12px; | |
| font-weight: 500; | |
| cursor: pointer; | |
| border-bottom: 2px solid transparent; | |
| transition: all .15s; | |
| text-align: center; | |
| } | |
| .mobile-tab.active { | |
| color: var(--accent); | |
| border-bottom-color: var(--accent); | |
| } | |
| .mobile-tab.tab-disabled { | |
| opacity: 0.3; | |
| pointer-events: none; | |
| } | |
| /* βββ Main Split βββββββββββββββββββββββββββββββββββββββ */ | |
| main { | |
| flex: 1; min-height: 0; overflow: hidden; | |
| display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 0; | |
| } | |
| main.single-panel { grid-template-columns: 1fr 2fr; } | |
| /* βββ Left Panel (Input) βββββββββββββββββββββββββββββββ */ | |
| .input-panel { | |
| border-right: 1px solid var(--border); | |
| display: flex; flex-direction: column; overflow: hidden; | |
| } | |
| .panel-header { | |
| padding: 7px 16px 7px; border-bottom: 1px solid var(--border); | |
| display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; | |
| } | |
| .panel-title { | |
| font-family: 'Bricolage Grotesque', sans-serif; font-size: 13px; font-weight: 600; | |
| color: var(--text2); letter-spacing: .3px; display: flex; align-items: center; gap: 6px; | |
| } | |
| .panel-title-icon { | |
| width: 18px; height: 18px; background: var(--bg4); border: 1px solid var(--border); | |
| border-radius: 4px; display: flex; align-items: center; justify-content: center; font-size: 10px; | |
| } | |
| .sample-btns { display: flex; gap: 4px; flex-wrap: wrap; } | |
| .sample-btn { | |
| font-size: 10px; padding: 3px 8px; border-radius: 5px; | |
| border: 1px solid var(--border); background: var(--bg2); color: var(--text2); | |
| cursor: pointer; font-family: 'DM Sans', sans-serif; transition: all .15s; | |
| } | |
| .sample-btn:hover { border-color: var(--border2); color: var(--text); } | |
| #input-area { | |
| flex: 1; width: 100%; background: transparent; border: none; outline: none; | |
| resize: none; color: var(--text); font-family: 'DM Sans', sans-serif; | |
| font-size: 14px; line-height: 1.7; padding: 14px 16px; min-height: 0; overflow-y: auto; | |
| } | |
| #input-area::placeholder { color: var(--text3); } | |
| .char-counter { | |
| padding: 6px 16px; border-top: 1px solid var(--border); flex-shrink: 0; | |
| font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); text-align: right; | |
| } | |
| /* βββ Output Panel βββββββββββββββββββββββββββββββββββββ */ | |
| .output-panel { | |
| display: flex; flex-direction: column; overflow: hidden; min-height: 0; | |
| } | |
| .output-panel + .output-panel { border-left: 1px solid var(--border); } | |
| .output-panel-header { | |
| padding: 10px 14px; border-bottom: 1px solid var(--border); | |
| display: flex; align-items: center; justify-content: space-between; flex-shrink: 0; gap: 8px; | |
| } | |
| .model-indicator { | |
| display: flex; align-items: center; gap: 5px; font-size: 11px; | |
| font-family: 'JetBrains Mono', monospace; color: var(--text2); min-width: 0; overflow: hidden; | |
| } | |
| .model-indicator-dot { width: 7px; height: 7px; border-radius: 50%; flex-shrink: 0; } | |
| .model-indicator-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } | |
| /* Stats row */ | |
| .stats-row { | |
| display: grid; grid-template-columns: 1fr 1fr; | |
| border-bottom: 1px solid var(--border); flex-shrink: 0; | |
| } | |
| .stat-card { | |
| padding: 10px 14px; border-right: 1px solid var(--border); | |
| border-bottom: 1px solid var(--border); position: relative; overflow: hidden; | |
| } | |
| .stat-card:nth-child(2n) { border-right: none; } | |
| .stat-card:nth-child(3), .stat-card:nth-child(4) { border-bottom: none; } | |
| .stat-card::after { | |
| content: ''; position: absolute; bottom: 0; left: 0; right: 0; height: 2px; | |
| background: linear-gradient(90deg, transparent, var(--accent), transparent); | |
| opacity: 0; transition: opacity .3s; | |
| } | |
| .stat-card.highlight::after { opacity: 1; } | |
| .stat-label { | |
| font-size: 9px; font-family: 'JetBrains Mono', monospace; color: var(--text3); | |
| text-transform: uppercase; letter-spacing: 1px; margin-bottom: 4px; | |
| } | |
| .stat-value { | |
| font-family: 'Bricolage Grotesque', sans-serif; font-size: 20px; font-weight: 700; | |
| color: var(--text); line-height: 1; transition: all .3s; | |
| } | |
| .stat-card:nth-child(1) .stat-value { color: var(--accent); } | |
| .stat-card:nth-child(2) .stat-value { color: var(--green); } | |
| .stat-card:nth-child(3) .stat-value { color: var(--amber); } | |
| .stat-card:nth-child(4) .stat-value { color: var(--accent2); } | |
| .stat-sub { | |
| font-size: 9px; color: var(--text3); font-family: 'JetBrains Mono', monospace; margin-top: 2px; | |
| } | |
| /* View toggle */ | |
| .view-toggle { | |
| display: flex; padding: 8px 14px; border-bottom: 1px solid var(--border); | |
| gap: 4px; align-items: center; justify-content: space-between; flex-shrink: 0; | |
| } | |
| .toggle-group { | |
| display: flex; gap: 2px; background: var(--bg2); border: 1px solid var(--border); | |
| border-radius: 6px; padding: 2px; | |
| } | |
| .toggle-btn { | |
| padding: 3px 10px; border-radius: 4px; border: none; background: transparent; | |
| color: var(--text2); font-family: 'DM Sans', sans-serif; font-size: 11px; | |
| font-weight: 500; cursor: pointer; transition: all .15s; | |
| } | |
| .toggle-btn.active { background: var(--bg4); color: var(--text); box-shadow: 0 1px 4px rgba(0,0,0,.3); } | |
| /* Token Display */ | |
| .token-display { | |
| flex: 1; overflow-y: auto; padding: 14px; min-height: 0; | |
| scrollbar-width: thin; scrollbar-color: var(--border) transparent; | |
| } | |
| .placeholder-msg { | |
| display: flex; flex-direction: column; align-items: center; justify-content: center; | |
| height: 100%; min-height: 120px; gap: 12px; color: var(--text3); | |
| } | |
| .placeholder-icon { font-size: 32px; filter: grayscale(1) opacity(.3); } | |
| .placeholder-msg p { | |
| font-family: 'JetBrains Mono', monospace; font-size: 11px; text-align: center; line-height: 1.6; | |
| } | |
| /* βββ Token Visualization Views βββββββββββββββββββββββββ */ | |
| .token-text-view { | |
| font-family: 'JetBrains Mono', monospace; font-size: 13px; | |
| line-height: 2.2; word-break: break-all; | |
| } | |
| .tok { | |
| display: inline; border-radius: 3px; padding: 1px 0; | |
| cursor: default; transition: filter .15s; position: relative; | |
| } | |
| .tok:hover { filter: brightness(1.3); } | |
| .tok-tooltip { | |
| display: none; position: absolute; bottom: 110%; left: 50%; | |
| transform: translateX(-50%); background: var(--bg4); border: 1px solid var(--border2); | |
| border-radius: 5px; padding: 4px 7px; font-size: 10px; white-space: nowrap; | |
| z-index: 50; pointer-events: none; box-shadow: 0 4px 20px rgba(0,0,0,.5); | |
| } | |
| [data-theme="light"] .tok-tooltip { box-shadow: 0 4px 16px rgba(0,0,0,.12); } | |
| .tok:hover .tok-tooltip { display: block; } | |
| .tok-tooltip-id { color: var(--accent); font-weight: 700; } | |
| .tok-tooltip-text { color: var(--text2); } | |
| .tok-space::before { content: 'Β·'; opacity: .3; } | |
| .tok-newline::before { content: 'β΅'; opacity: .5; } | |
| /* ID VIEW */ | |
| .token-id-view { display: flex; flex-wrap: wrap; gap: 5px; } | |
| .tok-id-card { | |
| display: flex; flex-direction: column; align-items: center; border-radius: 6px; | |
| overflow: hidden; border: 1px solid; cursor: default; | |
| transition: transform .15s, box-shadow .15s; min-width: 46px; | |
| } | |
| .tok-id-card:hover { transform: translateY(-2px); box-shadow: 0 4px 16px rgba(0,0,0,.4); } | |
| .tok-id-top { | |
| padding: 2px 5px; font-family: 'JetBrains Mono', monospace; font-size: 10px; | |
| font-weight: 500; width: 100%; text-align: center; border-bottom: 1px solid rgba(255,255,255,.08); | |
| } | |
| [data-theme="light"] .tok-id-top { border-bottom-color: rgba(0,0,0,.06); } | |
| .tok-id-bottom { | |
| padding: 1px 5px 2px; font-family: 'JetBrains Mono', monospace; font-size: 8px; | |
| color: rgba(255,255,255,.4); width: 100%; text-align: center; | |
| } | |
| [data-theme="light"] .tok-id-bottom { color: rgba(0,0,0,.35); } | |
| /* LIST VIEW */ | |
| .token-split-view { display: flex; flex-direction: column; gap: 2px; } | |
| .tok-split-row { | |
| display: flex; align-items: stretch; border-radius: 5px; overflow: hidden; | |
| border: 1px solid; font-family: 'JetBrains Mono', monospace; font-size: 11px; | |
| } | |
| .tok-split-idx { | |
| width: 34px; text-align: center; padding: 4px 3px; font-size: 9px; | |
| color: rgba(255,255,255,.3); border-right: 1px solid rgba(255,255,255,.06); | |
| display: flex; align-items: center; justify-content: center; | |
| } | |
| [data-theme="light"] .tok-split-idx { color: rgba(0,0,0,.25); border-right-color: rgba(0,0,0,.06); } | |
| .tok-split-text { flex: 1; padding: 4px 6px; font-size: 12px; } | |
| .tok-split-id { | |
| padding: 4px 6px; font-size: 10px; color: rgba(255,255,255,.45); | |
| border-left: 1px solid rgba(255,255,255,.06); display: flex; align-items: center; | |
| } | |
| [data-theme="light"] .tok-split-id { color: rgba(0,0,0,.35); border-left-color: rgba(0,0,0,.06); } | |
| /* βββ Loading Overlay ββββββββββββββββββββββββββββββββββ */ | |
| #loading-overlay { | |
| position: fixed; inset: 0; background: rgba(6,11,20,.92); | |
| backdrop-filter: blur(8px); z-index: 1000; | |
| display: flex; flex-direction: column; align-items: center; justify-content: center; | |
| gap: 20px; transition: opacity .4s; | |
| } | |
| [data-theme="light"] #loading-overlay { background: rgba(238,241,248,.92); } | |
| #loading-overlay.hidden { opacity: 0; pointer-events: none; } | |
| .loading-spinner { width: 48px; height: 48px; position: relative; } | |
| .loading-spinner::before, .loading-spinner::after { | |
| content: ''; position: absolute; border-radius: 50%; border: 2px solid transparent; | |
| } | |
| .loading-spinner::before { inset: 0; border-top-color: var(--accent); animation: spin 1s linear infinite; } | |
| .loading-spinner::after { inset: 7px; border-top-color: var(--accent2); animation: spin .7s linear infinite reverse; } | |
| @keyframes spin { to { transform: rotate(360deg); } } | |
| .loading-text { font-family: 'Bricolage Grotesque', sans-serif; font-size: 18px; font-weight: 600; color: var(--text); } | |
| .loading-sub { | |
| font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--text2); | |
| max-width: 340px; text-align: center; | |
| } | |
| .loading-bar-wrap { width: 260px; height: 3px; background: var(--bg3); border-radius: 2px; overflow: hidden; } | |
| .loading-bar { | |
| height: 100%; width: 0%; | |
| background: linear-gradient(90deg, var(--accent), var(--accent2)); | |
| border-radius: 2px; transition: width .3s; | |
| } | |
| .loading-file { font-size: 10px; font-family: 'JetBrains Mono', monospace; color: var(--text3); } | |
| /* βββ Error Toast ββββββββββββββββββββββββββββββββββββββ */ | |
| #toast { | |
| position: fixed; bottom: 24px; left: 50%; | |
| transform: translateX(-50%) translateY(80px); | |
| background: rgba(245,85,119,.15); border: 1px solid rgba(245,85,119,.4); | |
| color: var(--red); padding: 8px 18px; border-radius: 8px; | |
| font-size: 12px; font-family: 'JetBrains Mono', monospace; | |
| z-index: 500; transition: transform .3s; max-width: 460px; text-align: center; | |
| } | |
| #toast.show { transform: translateX(-50%) translateY(0); } | |
| /* βββ Footer βββββββββββββββββββββββββββββββββββββββββββ */ | |
| footer { | |
| padding: 8px 24px; border-top: 1px solid var(--border); | |
| display: flex; align-items: center; justify-content: space-between; | |
| font-size: 10px; color: var(--text3); font-family: 'JetBrains Mono', monospace; | |
| background: rgba(6,11,20,.8); flex-shrink: 0; | |
| } | |
| [data-theme="light"] footer { background: rgba(238,241,248,.8); } | |
| footer a { color: var(--text2); text-decoration: none; transition: color .15s; } | |
| footer a:hover { color: var(--accent); } | |
| /* βββ Scrollbar ββββββββββββββββββββββββββββββββββββββββ */ | |
| ::-webkit-scrollbar { width: 5px; } | |
| ::-webkit-scrollbar-track { background: transparent; } | |
| ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } | |
| ::-webkit-scrollbar-thumb:hover { background: var(--border2); } | |
| /* βββ Animations βββββββββββββββββββββββββββββββββββββββ */ | |
| @keyframes fadeIn { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: translateY(0); } } | |
| .fade-in { animation: fadeIn .2s ease forwards; } | |
| /* βββ Responsive: Tablet (600β900px) ββββββββββββββββββββ */ | |
| @media (min-width: 601px) and (max-width: 900px) { | |
| header { | |
| flex-wrap: wrap; | |
| height: auto; | |
| padding: 8px 12px; | |
| gap: 8px; | |
| } | |
| .header-divider { display: none; } | |
| .header-controls { | |
| flex-basis: 100%; | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 8px; | |
| } | |
| .searchbar-group { | |
| flex: 1 1 180px; | |
| } | |
| main { | |
| grid-template-columns: 1fr; | |
| grid-template-rows: auto 1fr 1fr; | |
| } | |
| main.single-panel { | |
| grid-template-columns: 1fr; | |
| grid-template-rows: auto 1fr; | |
| } | |
| .input-panel { | |
| border-right: none; | |
| border-bottom: 1px solid var(--border); | |
| max-height: 25vh; | |
| } | |
| .output-panel { | |
| border-left: none ; | |
| border-bottom: 1px solid var(--border); | |
| } | |
| .output-panel:last-child { | |
| border-bottom: none; | |
| } | |
| .stat-card { padding: 7px 10px; } | |
| .stat-value { font-size: 17px; } | |
| .stat-sub { font-size: 8px; } | |
| .stat-label { font-size: 8px; } | |
| .output-panel-header { padding: 8px 10px; } | |
| .view-toggle { padding: 6px 10px; } | |
| } | |
| /* βββ Responsive: Mobile (β€600px) ββββββββββββββββββββββ */ | |
| @media (max-width: 600px) { | |
| header { | |
| flex-wrap: wrap; | |
| height: auto; | |
| padding: 6px 10px; | |
| gap: 6px; | |
| } | |
| .header-divider { display: none; } | |
| .logo-tag { display: none; } | |
| .header-controls { | |
| flex-basis: 100%; | |
| display: flex; | |
| flex-wrap: wrap; | |
| gap: 6px; | |
| } | |
| .searchbar-group { | |
| flex: 1 1 100%; | |
| } | |
| .searchbar-label { display: none; } | |
| .dropdown-menu { | |
| left: 0 ; | |
| right: 0 ; | |
| min-width: unset; | |
| } | |
| .mobile-tab-bar { | |
| display: flex; | |
| } | |
| main { | |
| grid-template-columns: 1fr; | |
| grid-template-rows: 1fr; | |
| } | |
| .input-panel, | |
| .output-panel { | |
| display: none; | |
| } | |
| .mobile-active { | |
| display: flex ; | |
| } | |
| .input-panel { | |
| border-right: none; | |
| border-bottom: none; | |
| max-height: none; | |
| } | |
| .output-panel { | |
| border-left: none ; | |
| border-bottom: none; | |
| } | |
| .panel-header { | |
| flex-wrap: wrap; | |
| gap: 6px; | |
| padding: 10px 12px 8px; | |
| } | |
| .sample-btns { width: 100%; } | |
| .sample-btn { font-size: 9px; padding: 3px 6px; } | |
| .output-panel-header { padding: 6px 10px; } | |
| .model-indicator { font-size: 10px; } | |
| .stat-card { padding: 5px 8px; } | |
| .stat-value { font-size: 16px; } | |
| .stat-label { font-size: 7px; letter-spacing: .5px; margin-bottom: 2px; } | |
| .stat-sub { font-size: 7px; } | |
| .view-toggle { padding: 5px 10px; } | |
| .toggle-btn { padding: 3px 8px; font-size: 10px; } | |
| .token-display { padding: 10px; } | |
| .token-text-view { font-size: 12px; line-height: 2; } | |
| .tok-split-idx { width: 28px; font-size: 8px; } | |
| .tok-split-text { font-size: 11px; } | |
| .tok-split-id { font-size: 9px; } | |
| footer { | |
| padding: 6px 12px; | |
| font-size: 9px; | |
| } | |
| footer span:last-child { display: none; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="bg-gradient"></div> | |
| <div class="dot-grid"></div> | |
| <div id="app"> | |
| <!-- Header --> | |
| <header> | |
| <div class="logo"> | |
| <div class="logo-hex">T</div> | |
| <span class="logo-name">TokenLens</span> | |
| <span class="logo-tag">v1.4</span> | |
| </div> | |
| <div class="header-divider"></div> | |
| <div class="header-controls"> | |
| <!-- Search bar A --> | |
| <div class="searchbar-group" id="search-group-0"> | |
| <span class="searchbar-label label-a">A</span> | |
| <input class="searchbar-input" id="search-input-0" type="text" placeholder="Model A: HF repo idβ¦" /> | |
| <button class="searchbar-dropdown-btn" id="dropdown-btn-0" title="Predefined models"> | |
| <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg> | |
| </button> | |
| <button class="searchbar-load-btn" id="load-btn-0">Load</button> | |
| <div class="dropdown-menu" id="dropdown-menu-0"></div> | |
| </div> | |
| <!-- Search bar B --> | |
| <div class="searchbar-group" id="search-group-1"> | |
| <span class="searchbar-label label-b">B</span> | |
| <input class="searchbar-input" id="search-input-1" type="text" placeholder="Model B: HF repo idβ¦" /> | |
| <button class="searchbar-dropdown-btn" id="dropdown-btn-1" title="Predefined models"> | |
| <svg width="10" height="10" viewBox="0 0 10 10" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M2 3.5L5 6.5L8 3.5"/></svg> | |
| </button> | |
| <button class="searchbar-load-btn" id="load-btn-1">Load</button> | |
| <div class="dropdown-menu" id="dropdown-menu-1"></div> | |
| </div> | |
| <!-- Toggle: show/hide panel B --> | |
| <button class="icon-toggle-btn active" id="panel-toggle" title="Toggle comparison panel"> | |
| <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><rect x="3" y="3" width="7" height="18" rx="1"/><rect x="14" y="3" width="7" height="18" rx="1"/></svg> | |
| </button> | |
| <!-- Toggle: light/dark theme --> | |
| <button class="icon-toggle-btn" id="theme-toggle" title="Toggle light/dark theme"> | |
| <svg id="theme-icon-sun" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" style="display:none"><circle cx="12" cy="12" r="5"/><line x1="12" y1="1" x2="12" y2="3"/><line x1="12" y1="21" x2="12" y2="23"/><line x1="4.22" y1="4.22" x2="5.64" y2="5.64"/><line x1="18.36" y1="18.36" x2="19.78" y2="19.78"/><line x1="1" y1="12" x2="3" y2="12"/><line x1="21" y1="12" x2="23" y2="12"/><line x1="4.22" y1="19.78" x2="5.64" y2="18.36"/><line x1="18.36" y1="5.64" x2="19.78" y2="4.22"/></svg> | |
| <svg id="theme-icon-moon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"/></svg> | |
| </button> | |
| </div> | |
| </header> | |
| <!-- Mobile Tab Bar --> | |
| <div class="mobile-tab-bar" id="mobile-tab-bar"> | |
| <button class="mobile-tab active" data-tab="input">Input</button> | |
| <button class="mobile-tab" data-tab="panel-0">Model A</button> | |
| <button class="mobile-tab" data-tab="panel-1">Model B</button> | |
| </div> | |
| <!-- Main --> | |
| <main id="main-grid"> | |
| <!-- Left: Input --> | |
| <div class="input-panel mobile-active" id="input-panel"> | |
| <div class="panel-header"> | |
| <div class="panel-title"> | |
| <div class="panel-title-icon">β</div> | |
| Input Text | |
| </div> | |
| <div class="sample-btns"> | |
| <button class="sample-btn" data-sample="poetry">Poetry</button> | |
| <button class="sample-btn" data-sample="code">Code</button> | |
| <button class="sample-btn" data-sample="multilingual">Multi</button> | |
| <button class="sample-btn" data-sample="numbers">Numbers</button> | |
| <button class="sample-btn" data-sample="clear">Clear</button> | |
| </div> | |
| </div> | |
| <textarea id="input-area" | |
| placeholder="Type or paste text here to see how tokenizers split it into tokens⦠| |
| Try special characters, code, emojis, or multi-lingual text to compare models."></textarea> | |
| <div class="char-counter"><span id="char-count">0</span> characters</div> | |
| </div> | |
| <!-- Visualizer A --> | |
| <div class="output-panel" id="panel-0"> | |
| <div class="output-panel-header"> | |
| <div class="model-indicator" id="model-indicator-0"> | |
| <div class="model-indicator-dot" id="model-dot-0" style="background:#3d5a80"></div> | |
| <span class="model-indicator-name" id="model-label-0">A: no model</span> | |
| </div> | |
| </div> | |
| <div class="stats-row"> | |
| <div class="stat-card" id="sc-tokens-0"> | |
| <div class="stat-label">Tokens</div> | |
| <div class="stat-value" id="stat-tokens-0">β</div> | |
| <div class="stat-sub" id="stat-model-0">no model loaded</div> | |
| </div> | |
| <div class="stat-card" id="sc-chars-0"> | |
| <div class="stat-label">Characters</div> | |
| <div class="stat-value" id="stat-chars-0">β</div> | |
| <div class="stat-sub">total input</div> | |
| </div> | |
| <div class="stat-card" id="sc-words-0"> | |
| <div class="stat-label">Words</div> | |
| <div class="stat-value" id="stat-words-0">β</div> | |
| <div class="stat-sub">approx</div> | |
| </div> | |
| <div class="stat-card" id="sc-ratio-0"> | |
| <div class="stat-label">Chars/Token</div> | |
| <div class="stat-value" id="stat-ratio-0">β</div> | |
| <div class="stat-sub">efficiency</div> | |
| </div> | |
| </div> | |
| <div class="view-toggle"> | |
| <div class="toggle-group" id="toggle-group-0"> | |
| <button class="toggle-btn active" data-view="text" data-panel="0">Text View</button> | |
| <button class="toggle-btn" data-view="ids" data-panel="0">ID Grid</button> | |
| <button class="toggle-btn" data-view="list" data-panel="0">Token List</button> | |
| </div> | |
| </div> | |
| <div class="token-display" id="token-display-0"> | |
| <div class="placeholder-msg" id="placeholder-0"> | |
| <div class="placeholder-icon">⬑</div> | |
| <p>Load a tokenizer using search bar A above<br>then type text to see tokenization</p> | |
| </div> | |
| </div> | |
| </div> | |
| <!-- Visualizer B --> | |
| <div class="output-panel" id="panel-1"> | |
| <div class="output-panel-header"> | |
| <div class="model-indicator" id="model-indicator-1"> | |
| <div class="model-indicator-dot" id="model-dot-1" style="background:#3d5a80"></div> | |
| <span class="model-indicator-name" id="model-label-1">B: no model</span> | |
| </div> | |
| </div> | |
| <div class="stats-row"> | |
| <div class="stat-card" id="sc-tokens-1"> | |
| <div class="stat-label">Tokens</div> | |
| <div class="stat-value" id="stat-tokens-1">β</div> | |
| <div class="stat-sub" id="stat-model-1">no model loaded</div> | |
| </div> | |
| <div class="stat-card" id="sc-chars-1"> | |
| <div class="stat-label">Characters</div> | |
| <div class="stat-value" id="stat-chars-1">β</div> | |
| <div class="stat-sub">total input</div> | |
| </div> | |
| <div class="stat-card" id="sc-words-1"> | |
| <div class="stat-label">Words</div> | |
| <div class="stat-value" id="stat-words-1">β</div> | |
| <div class="stat-sub">approx</div> | |
| </div> | |
| <div class="stat-card" id="sc-ratio-1"> | |
| <div class="stat-label">Chars/Token</div> | |
| <div class="stat-value" id="stat-ratio-1">β</div> | |
| <div class="stat-sub">efficiency</div> | |
| </div> | |
| </div> | |
| <div class="view-toggle"> | |
| <div class="toggle-group" id="toggle-group-1"> | |
| <button class="toggle-btn active" data-view="text" data-panel="1">Text View</button> | |
| <button class="toggle-btn" data-view="ids" data-panel="1">ID Grid</button> | |
| <button class="toggle-btn" data-view="list" data-panel="1">Token List</button> | |
| </div> | |
| </div> | |
| <div class="token-display" id="token-display-1"> | |
| <div class="placeholder-msg" id="placeholder-1"> | |
| <div class="placeholder-icon">⬑</div> | |
| <p>Load a tokenizer using search bar B above<br>then type text to see tokenization</p> | |
| </div> | |
| </div> | |
| </div> | |
| </main> | |
| <footer> | |
| <span>TokenLens β Powered by <a href="https://github.com/xenova/transformers.js" target="_blank">Transformers.js</a> Β· Runs entirely in your browser</span> | |
| <span><a href="https://quickgrid.github.io/">Made by Β· Asif Ahmed</a></span> | |
| </footer> | |
| </div> | |
| <!-- Loading Overlay --> | |
| <div id="loading-overlay"> | |
| <div class="loading-spinner"></div> | |
| <div class="loading-text" id="loading-title">Loading Tokenizer</div> | |
| <div class="loading-sub" id="loading-sub">Downloading tokenizer files from Hugging Face Hubβ¦<br>Cached in your browser after first download.</div> | |
| <div class="loading-bar-wrap"><div class="loading-bar" id="loading-bar"></div></div> | |
| <div class="loading-file" id="loading-file"></div> | |
| </div> | |
| <!-- Toast --> | |
| <div id="toast"></div> | |
| <script type="module"> | |
| import { AutoTokenizer, env } | |
| from 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@4.2.0'; | |
| env.allowLocalModels = false; | |
| env.useBrowserCache = true; | |
| // ββ Model Registry βββββββββββββββββββββββββββββββββββββββββ | |
| const MODELS = [ | |
| { | |
| id:'Qwen/Qwen3.6-27B', | |
| name:'Qwen 3.6 27B', | |
| org:'Alibaba', | |
| color:'#0466de', | |
| vocab:'152k', | |
| type:'BPE', | |
| desc:'Qwen tokenizer optimized for multilingual coding and reasoning tasks' | |
| }, | |
| { | |
| id:'deepseek-ai/DeepSeek-V4-Pro', | |
| name:'DeepSeek V4 Pro', | |
| org:'DeepSeek', | |
| color:'#4285f4', | |
| vocab:'129k', | |
| type:'BPE', | |
| desc:'DeepSeek multilingual tokenizer designed for code-heavy workloads' | |
| }, | |
| { | |
| id:'MiniMaxAI/MiniMax-M2.7', | |
| name:'MiniMax M2.7', | |
| org:'MiniMax', | |
| color:'#1a73e8', | |
| vocab:'128k', | |
| type:'SentencePiece', | |
| desc:'Efficient multilingual tokenizer for long-context multimodal models' | |
| }, | |
| { | |
| id:'mistralai/Mistral-Medium-3.5-128B', | |
| name:'Mistral Medium 3.5', | |
| org:'Mistral AI', | |
| color:'#ff7722', | |
| vocab:'131k', | |
| type:'Tekken BPE', | |
| desc:'Mistral Tekken tokenizer with efficient multilingual compression' | |
| }, | |
| { | |
| id:'google/gemma-4-31B-it', | |
| name:'Gemma 4 31B', | |
| org:'Google', | |
| color:'#34a853', | |
| vocab:'256k', | |
| type:'SentencePiece', | |
| desc:'Gemma multilingual SentencePiece tokenizer with large vocabulary' | |
| }, | |
| { | |
| id:'zai-org/GLM-5.1', | |
| name:'GLM-5.1', | |
| org:'Z.ai', | |
| color:'#10a37f', | |
| vocab:'151k', | |
| type:'SentencePiece', | |
| desc:'GLM multilingual SentencePiece tokenizer with large bilingual vocabulary' | |
| }, | |
| { | |
| id:'XiaomiMiMo/MiMo-V2.5-Pro', | |
| name:'MiMo V2.5 Pro', | |
| org:'Xiaomi', | |
| color:'#ff7722', | |
| vocab:'128k', | |
| type:'SentencePiece', | |
| desc:'MoE tokenizer tuned for multilingual reasoning and code generation' | |
| }, | |
| ]; | |
| // ββ Token Color Palettes βββββββββββββββββββββββββββββββββββ | |
| const PALETTE_DARK = [ | |
| { text:'#ff8080', bg:'rgba(255,128,128,.18)', border:'rgba(255,128,128,.35)' }, | |
| { text:'#ffb84d', bg:'rgba(255,184, 77,.18)', border:'rgba(255,184, 77,.35)' }, | |
| { text:'#ffe066', bg:'rgba(255,224,102,.18)', border:'rgba(255,224,102,.35)' }, | |
| { text:'#7aed91', bg:'rgba(122,237,145,.18)', border:'rgba(122,237,145,.35)' }, | |
| { text:'#4ddfc0', bg:'rgba( 77,223,192,.18)', border:'rgba( 77,223,192,.35)' }, | |
| { text:'#56c8f5', bg:'rgba( 86,200,245,.18)', border:'rgba( 86,200,245,.35)' }, | |
| { text:'#748ef8', bg:'rgba(116,142,248,.18)', border:'rgba(116,142,248,.35)' }, | |
| { text:'#c484f8', bg:'rgba(196,132,248,.18)', border:'rgba(196,132,248,.35)' }, | |
| { text:'#f57cd4', bg:'rgba(245,124,212,.18)', border:'rgba(245,124,212,.35)' }, | |
| { text:'#fa8072', bg:'rgba(250,128,114,.18)', border:'rgba(250,128,114,.35)' }, | |
| { text:'#8be08b', bg:'rgba(139,224,139,.18)', border:'rgba(139,224,139,.35)' }, | |
| { text:'#f0c040', bg:'rgba(240,192, 64,.18)', border:'rgba(240,192, 64,.35)' }, | |
| { text:'#60d4e0', bg:'rgba( 96,212,224,.18)', border:'rgba( 96,212,224,.35)' }, | |
| { text:'#e89060', bg:'rgba(232,144, 96,.18)', border:'rgba(232,144, 96,.35)' }, | |
| ]; | |
| const PALETTE_LIGHT = [ | |
| { text:'#cc3333', bg:'rgba(204, 51, 51,.12)', border:'rgba(204, 51, 51,.25)' }, | |
| { text:'#b87218', bg:'rgba(184,114, 24,.12)', border:'rgba(184,114, 24,.25)' }, | |
| { text:'#a08618', bg:'rgba(160,134, 24,.12)', border:'rgba(160,134, 24,.25)' }, | |
| { text:'#228838', bg:'rgba( 34,136, 56,.12)', border:'rgba( 34,136, 56,.25)' }, | |
| { text:'#1a8870', bg:'rgba( 26,136,112,.12)', border:'rgba( 26,136,112,.25)' }, | |
| { text:'#1890b8', bg:'rgba( 24,144,184,.12)', border:'rgba( 24,144,184,.25)' }, | |
| { text:'#3850b8', bg:'rgba( 56, 80,184,.12)', border:'rgba( 56, 80,184,.25)' }, | |
| { text:'#7830a8', bg:'rgba(120, 48,168,.12)', border:'rgba(120, 48,168,.25)' }, | |
| { text:'#b03088', bg:'rgba(176, 48,136,.12)', border:'rgba(176, 48,136,.25)' }, | |
| { text:'#b83828', bg:'rgba(184, 56, 40,.12)', border:'rgba(184, 56, 40,.25)' }, | |
| { text:'#2a882a', bg:'rgba( 42,136, 42,.12)', border:'rgba( 42,136, 42,.25)' }, | |
| { text:'#9a7018', bg:'rgba(154,112, 24,.12)', border:'rgba(154,112, 24,.25)' }, | |
| { text:'#1a8898', bg:'rgba( 26,136,152,.12)', border:'rgba( 26,136,152,.25)' }, | |
| { text:'#a05020', bg:'rgba(160, 80, 32,.12)', border:'rgba(160, 80, 32,.25)' }, | |
| ]; | |
| function getPalette() { | |
| return document.documentElement.dataset.theme === 'light' ? PALETTE_LIGHT : PALETTE_DARK; | |
| } | |
| // ββ Sample texts βββββββββββββββββββββββββββββββββββββββββββ | |
| const SAMPLES = { | |
| poetry: `Two roads diverged in a yellow wood,\nAnd sorry I could not travel both\nAnd be one traveler, long I stood\nAnd looked down one as far as I could\nTo where it bent in the undergrowth;\nβ Robert Frost, "The Road Not Taken"`, | |
| code: `async function fetchData(url, retries = 3) {\n for (let i = 0; i < retries; i++) {\n try {\n const res = await fetch(url);\n if (!res.ok) throw new Error(\`HTTP \${res.status}\`);\n return await res.json();\n } catch (e) {\n if (i === retries - 1) throw e;\n await new Promise(r => setTimeout(r, 1000 * 2 ** i));\n }\n }\n}`, | |
| multilingual: `English: The quick brown fox jumps over the lazy dog.\nζ₯ζ¬θͺ: εΎθΌ©γ―η«γ§γγγεεγ―γΎγ γͺγγ\nδΈζ: ζ₯η δΈθ§ζοΌε€ε€ι»εΌιΈγ\nΨ§ΩΨΉΨ±Ψ¨ΩΨ©: Ψ§ΩΩΨΊΨ© Ψ§ΩΨΉΨ±Ψ¨ΩΨ© Ψ¬Ω ΩΩΨ© ΩΩ ΨΉΨ¨Ψ±Ψ©.\nΞλληνικά: Ξ Ξ³Ξ½ΟΟΞ· Ρίναι Ξ΄ΟΞ½Ξ±ΞΌΞ·.\nEmoji: π π¦ β‘ π― 𧬠π€ π¦`, | |
| numbers: `Ο β 3.14159265358979323846\ne β 2.71828182845904523536\nΟ β 1.61803398874989484820\n1,000,000 Γ $42.99 = $42,990,000.00\n2024-01-15T08:30:00.000Z\nIPv4: 192.168.1.1 | IPv6: ::1`, | |
| clear: '' | |
| }; | |
| // ββ State ββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| const panels = [ | |
| { tokenizer: null, modelId: null, view: 'text' }, | |
| { tokenizer: null, modelId: null, view: 'text' }, | |
| ]; | |
| let tokenizerCache = {}; | |
| let panel1Visible = true; | |
| let debounceTimer = null; | |
| let mobileActiveTab = 'input'; | |
| // ββ DOM References βββββββββββββββββββββββββββββββββββββββββ | |
| const $overlay = document.getElementById('loading-overlay'); | |
| const $loadTitle = document.getElementById('loading-title'); | |
| const $loadSub = document.getElementById('loading-sub'); | |
| const $loadBar = document.getElementById('loading-bar'); | |
| const $loadFile = document.getElementById('loading-file'); | |
| const $input = document.getElementById('input-area'); | |
| const $charCount = document.getElementById('char-count'); | |
| const $toast = document.getElementById('toast'); | |
| const $mainGrid = document.getElementById('main-grid'); | |
| const $panelToggle = document.getElementById('panel-toggle'); | |
| const $themeToggle = document.getElementById('theme-toggle'); | |
| const $inputPanel = document.getElementById('input-panel'); | |
| const $panel0 = document.getElementById('panel-0'); | |
| const $panel1 = document.getElementById('panel-1'); | |
| const $mobileTabs = document.querySelectorAll('.mobile-tab'); | |
| // ββ Utilities ββββββββββββββββββββββββββββββββββββββββββββββ | |
| function showOverlay(title, sub) { | |
| $loadTitle.textContent = title; | |
| $loadSub.textContent = sub; | |
| $loadBar.style.width = '0%'; | |
| $loadFile.textContent = ''; | |
| $overlay.classList.remove('hidden'); | |
| } | |
| function hideOverlay() { $overlay.classList.add('hidden'); } | |
| function showToast(msg, duration = 5000) { | |
| $toast.textContent = msg; | |
| $toast.classList.add('show'); | |
| setTimeout(() => $toast.classList.remove('show'), duration); | |
| } | |
| function setStats(idx, tokens, text) { | |
| const chars = text.length; | |
| const words = text.trim() ? text.trim().split(/\s+/).length : 0; | |
| const ratio = tokens > 0 && chars > 0 ? (chars / tokens).toFixed(2) : 'β'; | |
| document.getElementById(`stat-tokens-${idx}`).textContent = tokens > 0 ? tokens.toLocaleString() : 'β'; | |
| document.getElementById(`stat-chars-${idx}`).textContent = chars > 0 ? chars.toLocaleString() : 'β'; | |
| document.getElementById(`stat-words-${idx}`).textContent = words > 0 ? words.toLocaleString() : 'β'; | |
| document.getElementById(`stat-ratio-${idx}`).textContent = ratio; | |
| ['tokens','chars','words','ratio'].forEach(k => { | |
| const el = document.getElementById(`sc-${k}-${idx}`); | |
| el.classList.remove('highlight'); void el.offsetWidth; el.classList.add('highlight'); | |
| }); | |
| } | |
| function updateModelIndicator(idx, modelId) { | |
| const preset = MODELS.find(m => m.id === modelId); | |
| const color = preset ? preset.color : '#7899c0'; | |
| const name = modelId ? modelId.split('/').pop() : 'no model'; | |
| const label = idx === 0 ? 'A' : 'B'; | |
| document.getElementById(`model-dot-${idx}`).style.background = color; | |
| document.getElementById(`model-dot-${idx}`).style.boxShadow = `0 0 6px ${color}`; | |
| document.getElementById(`model-label-${idx}`).textContent = `${label}: ${name}`; | |
| document.getElementById(`stat-model-${idx}`).textContent = preset ? `${preset.org} Β· ${preset.type} Β· ${preset.vocab} vocab` : modelId || 'no model loaded'; | |
| } | |
| // ββ Decode raw token string for display βββββββββββββββββββ | |
| function decodeTokenString(raw) { | |
| if (!raw) return ''; | |
| let s = raw.replace(/^Δ /, ' ').replace(/Δ /g, ' '); | |
| s = s.replace(/^β/, ' ').replace(/β/g, ' '); | |
| s = s.replace(/Δ/g, '\n'); | |
| s = s.replace(/\r/g, ''); | |
| s = s.replace(/<0x([0-9A-Fa-f]{2})>/g, (_, hex) => { | |
| const code = parseInt(hex, 16); | |
| return code < 128 ? String.fromCharCode(code) : `[0x${hex}]`; | |
| }); | |
| return s; | |
| } | |
| // ββ Mobile Tab Handling ββββββββββββββββββββββββββββββββββββ | |
| function isMobile() { return window.innerWidth <= 600; } | |
| function applyMobileTab(tabId) { | |
| mobileActiveTab = tabId; | |
| $mobileTabs.forEach(t => t.classList.toggle('active', t.dataset.tab === tabId)); | |
| // Remove mobile-active from all panels | |
| $inputPanel.classList.remove('mobile-active'); | |
| $panel0.classList.remove('mobile-active'); | |
| $panel1.classList.remove('mobile-active'); | |
| // Add to the target | |
| if (tabId === 'input') $inputPanel.classList.add('mobile-active'); | |
| if (tabId === 'panel-0') $panel0.classList.add('mobile-active'); | |
| if (tabId === 'panel-1') $panel1.classList.add('mobile-active'); | |
| } | |
| $mobileTabs.forEach(tab => { | |
| tab.addEventListener('click', () => { | |
| applyMobileTab(tab.dataset.tab); | |
| }); | |
| }); | |
| // Handle resize: reset display properties when switching between mobile/desktop | |
| function handleResize() { | |
| if (!isMobile()) { | |
| // Desktop/tablet: remove mobile-active, reset display for all panels | |
| $inputPanel.classList.remove('mobile-active'); | |
| $panel0.classList.remove('mobile-active'); | |
| $panel1.classList.remove('mobile-active'); | |
| $inputPanel.style.display = ''; | |
| $panel0.style.display = ''; | |
| $panel1.style.display = panel1Visible ? '' : 'none'; | |
| } else { | |
| // Mobile: apply mobile tab logic | |
| $inputPanel.style.display = ''; | |
| $panel0.style.display = ''; | |
| $panel1.style.display = ''; | |
| applyMobileTab(mobileActiveTab); | |
| } | |
| updateMobileTabBState(); | |
| } | |
| function updateMobileTabBState() { | |
| const $tabB = document.querySelector('.mobile-tab[data-tab="panel-1"]'); | |
| if ($tabB) { | |
| if (panel1Visible) { | |
| $tabB.classList.remove('tab-disabled'); | |
| } else { | |
| $tabB.classList.add('tab-disabled'); | |
| // If currently on tab B, switch away | |
| if (mobileActiveTab === 'panel-1') { | |
| applyMobileTab('panel-0'); | |
| } | |
| } | |
| } | |
| } | |
| window.addEventListener('resize', handleResize); | |
| // ββ Tokenize for a specific panel βββββββββββββββββββββββββ | |
| async function tokenizeForPanel(idx, text) { | |
| const p = panels[idx]; | |
| const $display = document.getElementById(`token-display-${idx}`); | |
| const $placeholder = document.getElementById(`placeholder-${idx}`); | |
| if (!p.tokenizer || !text.trim()) { | |
| const prevView = $display.querySelector('.token-view-container'); | |
| if (prevView) prevView.remove(); | |
| if ($placeholder) $placeholder.style.display = 'flex'; | |
| setStats(idx, 0, text); | |
| return; | |
| } | |
| try { | |
| if ($placeholder) $placeholder.style.display = 'none'; | |
| const encoded = await p.tokenizer(text, { add_special_tokens: false }); | |
| const ids = Array.from(encoded.input_ids.data); | |
| let rawTokens; | |
| try { rawTokens = p.tokenizer.model.convert_ids_to_tokens(ids); } | |
| catch { rawTokens = await Promise.all(ids.map(id => p.tokenizer.decode([id], { skip_special_tokens: false }))); } | |
| const tokens = ids.map((id, i) => ({ | |
| id, raw: rawTokens[i] || '', display: decodeTokenString(rawTokens[i] || ''), | |
| })); | |
| setStats(idx, tokens.length, text); | |
| renderView(idx, tokens); | |
| } catch (err) { | |
| console.error('Tokenization error:', err); | |
| showToast(`Panel ${idx === 0 ? 'A' : 'B'} error: ${err.message}`); | |
| } | |
| } | |
| // ββ Render Views βββββββββββββββββββββββββββββββββββββββββββ | |
| function renderView(idx, tokens) { | |
| const view = panels[idx].view; | |
| if (view === 'text') renderTextView(idx, tokens); | |
| else if (view === 'ids') renderIdView(idx, tokens); | |
| else if (view === 'list') renderListView(idx, tokens); | |
| } | |
| function renderTextView(idx, tokens) { | |
| const PALETTE = getPalette(); | |
| const $display = document.getElementById(`token-display-${idx}`); | |
| const container = document.createElement('div'); | |
| container.className = 'token-text-view token-view-container fade-in'; | |
| tokens.forEach((tok, i) => { | |
| const c = PALETTE[i % PALETTE.length]; | |
| const span = document.createElement('span'); | |
| span.className = 'tok'; | |
| span.style.background = c.bg; | |
| span.style.color = c.text; | |
| span.style.borderBottom = `2px solid ${c.border}`; | |
| const disp = tok.display; | |
| if (disp === ' ') span.innerHTML = ' '; | |
| else if (disp === '\n') span.innerHTML = 'β΅<br>'; | |
| else if (disp === '\t') span.innerHTML = 'β '; | |
| else span.textContent = disp; | |
| const tip = document.createElement('div'); | |
| tip.className = 'tok-tooltip'; | |
| const rawEsc = tok.raw.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>'); | |
| tip.innerHTML = `<span class="tok-tooltip-id">#${tok.id}</span> Β· <span class="tok-tooltip-text">${rawEsc || '(empty)'}</span>`; | |
| span.appendChild(tip); | |
| container.appendChild(span); | |
| }); | |
| const prevView = $display.querySelector('.token-view-container'); | |
| if (prevView) prevView.remove(); | |
| document.getElementById(`placeholder-${idx}`).style.display = 'none'; | |
| $display.appendChild(container); | |
| } | |
| function renderIdView(idx, tokens) { | |
| const PALETTE = getPalette(); | |
| const $display = document.getElementById(`token-display-${idx}`); | |
| const container = document.createElement('div'); | |
| container.className = 'token-id-view token-view-container fade-in'; | |
| tokens.forEach((tok, i) => { | |
| const c = PALETTE[i % PALETTE.length]; | |
| const card = document.createElement('div'); | |
| card.className = 'tok-id-card'; | |
| card.style.background = c.bg; | |
| card.style.borderColor = c.border; | |
| card.title = `Raw: ${tok.raw}`; | |
| const top = document.createElement('div'); | |
| top.className = 'tok-id-top'; top.style.color = c.text; top.textContent = tok.id; | |
| const bot = document.createElement('div'); | |
| bot.className = 'tok-id-bottom'; | |
| bot.textContent = tok.display.slice(0, 8).replace(/\n/g,'β΅').replace(/\t/g,'β') || 'β¦'; | |
| card.appendChild(top); card.appendChild(bot); | |
| container.appendChild(card); | |
| }); | |
| const prevView = $display.querySelector('.token-view-container'); | |
| if (prevView) prevView.remove(); | |
| document.getElementById(`placeholder-${idx}`).style.display = 'none'; | |
| $display.appendChild(container); | |
| } | |
| function renderListView(idx, tokens) { | |
| const PALETTE = getPalette(); | |
| const $display = document.getElementById(`token-display-${idx}`); | |
| const container = document.createElement('div'); | |
| container.className = 'token-split-view token-view-container fade-in'; | |
| tokens.forEach((tok, i) => { | |
| const c = PALETTE[i % PALETTE.length]; | |
| const row = document.createElement('div'); | |
| row.className = 'tok-split-row'; | |
| row.style.background = c.bg; | |
| row.style.borderColor = c.border; | |
| const idxEl = document.createElement('div'); | |
| idxEl.className = 'tok-split-idx'; idxEl.textContent = i; | |
| const textEl = document.createElement('div'); | |
| textEl.className = 'tok-split-text'; textEl.style.color = c.text; | |
| textEl.textContent = tok.display.replace(/\n/g,'β΅').replace(/\t/g,'β') || '(empty)'; | |
| const idEl = document.createElement('div'); | |
| idEl.className = 'tok-split-id'; idEl.textContent = tok.id; | |
| row.appendChild(idxEl); row.appendChild(textEl); row.appendChild(idEl); | |
| container.appendChild(row); | |
| }); | |
| const prevView = $display.querySelector('.token-view-container'); | |
| if (prevView) prevView.remove(); | |
| document.getElementById(`placeholder-${idx}`).style.display = 'none'; | |
| $display.appendChild(container); | |
| } | |
| // ββ Load Tokenizer βββββββββββββββββββββββββββββββββββββββββ | |
| async function loadModel(idx, modelId) { | |
| if (tokenizerCache[modelId]) { | |
| panels[idx].tokenizer = tokenizerCache[modelId]; | |
| panels[idx].modelId = modelId; | |
| updateModelIndicator(idx, modelId); | |
| await runTokenize(); | |
| return; | |
| } | |
| const displayName = modelId.split('/').pop(); | |
| const label = idx === 0 ? 'A' : 'B'; | |
| showOverlay( | |
| `Loading ${label}: ${displayName}`, | |
| `Fetching tokenizer files from Hugging Face Hub.\nCached in IndexedDB after first download.` | |
| ); | |
| let lastProgress = 0; | |
| try { | |
| const tokenizer = await AutoTokenizer.from_pretrained(modelId, { | |
| progress_callback: (info) => { | |
| if (info.status === 'downloading') { | |
| const pct = info.total ? Math.round((info.loaded / info.total) * 100) : lastProgress; | |
| $loadBar.style.width = pct + '%'; | |
| $loadFile.textContent = info.file || ''; | |
| lastProgress = pct; | |
| } else if (info.status === 'done') { | |
| $loadBar.style.width = '100%'; | |
| } | |
| } | |
| }); | |
| tokenizerCache[modelId] = tokenizer; | |
| panels[idx].tokenizer = tokenizer; | |
| panels[idx].modelId = modelId; | |
| updateModelIndicator(idx, modelId); | |
| hideOverlay(); | |
| await runTokenize(); | |
| } catch (err) { | |
| hideOverlay(); | |
| console.error('Failed to load tokenizer:', err); | |
| showToast(`Failed to load "${modelId}": ${err.message}`, 8000); | |
| } | |
| } | |
| // ββ Build Dropdown Menus βββββββββββββββββββββββββββββββββββ | |
| function buildDropdowns() { | |
| [0, 1].forEach(idx => { | |
| const $menu = document.getElementById(`dropdown-menu-${idx}`); | |
| $menu.innerHTML = ''; | |
| MODELS.forEach(m => { | |
| const item = document.createElement('div'); | |
| item.className = 'dropdown-item'; | |
| item.innerHTML = ` | |
| <span class="dropdown-item-dot" style="background:${m.color}"></span> | |
| <span class="dropdown-item-name">${m.name}</span> | |
| <span class="dropdown-item-detail">${m.org} Β· ${m.type}</span> | |
| `; | |
| item.addEventListener('click', () => { | |
| const $input = document.getElementById(`search-input-${idx}`); | |
| $input.value = m.id; | |
| $menu.classList.remove('open'); | |
| loadModel(idx, m.id); | |
| }); | |
| $menu.appendChild(item); | |
| }); | |
| }); | |
| } | |
| // ββ Dropdown toggle ββββββββββββββββββββββββββββββββββββββββ | |
| [0, 1].forEach(idx => { | |
| const $btn = document.getElementById(`dropdown-btn-${idx}`); | |
| const $menu = document.getElementById(`dropdown-menu-${idx}`); | |
| $btn.addEventListener('click', (e) => { | |
| e.stopPropagation(); | |
| const otherIdx = 1 - idx; | |
| document.getElementById(`dropdown-menu-${otherIdx}`).classList.remove('open'); | |
| $menu.classList.toggle('open'); | |
| }); | |
| }); | |
| document.addEventListener('click', () => { | |
| document.querySelectorAll('.dropdown-menu').forEach(m => m.classList.remove('open')); | |
| }); | |
| document.querySelectorAll('.dropdown-menu').forEach(m => { | |
| m.addEventListener('click', e => e.stopPropagation()); | |
| }); | |
| // ββ Load buttons βββββββββββββββββββββββββββββββββββββββββββ | |
| [0, 1].forEach(idx => { | |
| const $btn = document.getElementById(`load-btn-${idx}`); | |
| const $input = document.getElementById(`search-input-${idx}`); | |
| function doLoad() { | |
| const id = $input.value.trim(); | |
| if (!id) { showToast('Please enter a model ID'); return; } | |
| loadModel(idx, id); | |
| } | |
| $btn.addEventListener('click', doLoad); | |
| $input.addEventListener('keydown', e => { if (e.key === 'Enter') doLoad(); }); | |
| }); | |
| // ββ View Toggles βββββββββββββββββββββββββββββββββββββββββββ | |
| document.querySelectorAll('.toggle-btn').forEach(btn => { | |
| btn.addEventListener('click', () => { | |
| const panelIdx = parseInt(btn.dataset.panel); | |
| const group = document.getElementById(`toggle-group-${panelIdx}`); | |
| group.querySelectorAll('.toggle-btn').forEach(b => b.classList.remove('active')); | |
| btn.classList.add('active'); | |
| panels[panelIdx].view = btn.dataset.view; | |
| runTokenize(); | |
| }); | |
| }); | |
| // ββ Input Handling βββββββββββββββββββββββββββββββββββββββββ | |
| async function runTokenize() { | |
| const text = $input.value; | |
| $charCount.textContent = text.length; | |
| await Promise.all([ | |
| tokenizeForPanel(0, text), | |
| panel1Visible ? tokenizeForPanel(1, text) : Promise.resolve() | |
| ]); | |
| } | |
| $input.addEventListener('input', () => { | |
| $charCount.textContent = $input.value.length; | |
| clearTimeout(debounceTimer); | |
| debounceTimer = setTimeout(runTokenize, 280); | |
| }); | |
| // ββ Sample Buttons βββββββββββββββββββββββββββββββββββββββββ | |
| document.querySelectorAll('.sample-btn').forEach(btn => { | |
| btn.addEventListener('click', () => { | |
| $input.value = SAMPLES[btn.dataset.sample] ?? ''; | |
| $input.focus(); | |
| runTokenize(); | |
| }); | |
| }); | |
| // ββ Panel Toggle βββββββββββββββββββββββββββββββββββββββββββ | |
| $panelToggle.addEventListener('click', () => { | |
| panel1Visible = !panel1Visible; | |
| $panelToggle.classList.toggle('active', panel1Visible); | |
| if (!isMobile()) { | |
| $panel1.style.display = panel1Visible ? '' : 'none'; | |
| } | |
| const $searchGroup1 = document.getElementById('search-group-1'); | |
| $searchGroup1.style.display = panel1Visible ? '' : 'none'; | |
| if (panel1Visible) { | |
| $mainGrid.classList.remove('single-panel'); | |
| } else { | |
| $mainGrid.classList.add('single-panel'); | |
| } | |
| updateMobileTabBState(); | |
| runTokenize(); | |
| }); | |
| // ββ Theme Toggle βββββββββββββββββββββββββββββββββββββββββββ | |
| const $iconSun = document.getElementById('theme-icon-sun'); | |
| const $iconMoon = document.getElementById('theme-icon-moon'); | |
| function setTheme(theme) { | |
| document.documentElement.dataset.theme = theme; | |
| if (theme === 'light') { | |
| $iconSun.style.display = 'none'; | |
| $iconMoon.style.display = 'block'; | |
| } else { | |
| $iconSun.style.display = 'block'; | |
| $iconMoon.style.display = 'none'; | |
| } | |
| runTokenize(); | |
| } | |
| $themeToggle.addEventListener('click', () => { | |
| const current = document.documentElement.dataset.theme; | |
| setTheme(current === 'dark' ? 'light' : 'dark'); | |
| }); | |
| // ββ Init βββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| buildDropdowns(); | |
| $overlay.classList.add('hidden'); | |
| $input.value = ''; | |
| setTheme('dark'); | |
| // Set initial mobile state | |
| handleResize(); | |
| document.getElementById('search-input-0').value = MODELS[0].id; | |
| loadModel(0, MODELS[0].id); | |
| </script> | |
| </body> | |
| </html> |