tokenizer-arena / app.TabbedInterface.py
xu-song's picture
update gradio to 4.38.1
8167cc2
raw
history blame
887 Bytes
"""Gradio app to showcase the LLM tokenization.
"""
import os
from playground_app import demo as playground_tab
from compression_app import demo as compression_tab
from character_app import demo as character_tab
from patcher.gr_interface import TabbedInterface
from huggingface_hub import login
auth_token = os.environ.get('HF_TOKEN', None)
if auth_token:
login(token=auth_token)
# More Tab: encoding efficiency; decoding efficiency;
demo = TabbedInterface(
[playground_tab, compression_tab, character_tab],
[" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"],
title='<div align="center">Tokenizer Arena ⚔️</div>',
css="css/style.css"
)
demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
if __name__ == "__main__":
demo.queue(max_size=1024, default_concurrency_limit=80).launch()