File size: 248 Bytes
c57f5ec
 
 
 
 
 
 
1
2
3
4
5
6
7
8
import streamlit as st
from transformers import DistilBertTokenizerFast


@st.cache(allow_output_mutation=True)
def get_tokenizer(num_labels=153):
    return DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased', num_labels=num_labels)