|
"""Gradio app to showcase the LLM tokenization.
|
|
|
|
"""
|
|
|
|
import os
|
|
from playground_app import demo as playground_tab
|
|
from compression_app import demo as compression_tab
|
|
from character_app import demo as character_tab
|
|
from patcher.gr_interface import TabbedInterface
|
|
from huggingface_hub import login
|
|
|
|
auth_token = os.environ.get('HF_TOKEN', None)
|
|
if auth_token:
|
|
login(token=auth_token)
|
|
|
|
|
|
demo = TabbedInterface(
|
|
[playground_tab, compression_tab, character_tab],
|
|
[" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"],
|
|
title='<div align="center">Tokenizer Arena ⚔️</div>',
|
|
css="css/style.css"
|
|
)
|
|
|
|
demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
|
|
|
|
if __name__ == "__main__":
|
|
demo.queue(max_size=1024, default_concurrency_limit=80).launch()
|
|
|