tokenizer-arena / app.py
eson's picture
add access token
f1b4ae2
import os
from playground_app import demo as playground_tab
from compression_app import demo as compression_tab
from character_app import demo as character_tab
from patcher.gr_interface import TabbedInterface
from huggingface_hub import login
auth_token = os.environ.get('HF_TOKEN', None)
if auth_token:
login(token=auth_token)
# 编码速度,解码速度,字符分类(zh、num等,支持正则),支持的语言,。
demo = TabbedInterface(
[playground_tab, compression_tab, character_tab],
[" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"],
title='<div align="center">Tokenizer Arena ⚔️</div>',
css="css/style.css"
)
demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
if __name__ == "__main__":
demo.launch()