import streamlit as st from utils import get_res st.sidebar.title('Tokenizers demo') #x = st.slider('Select a value') #st.write(x, 'squared is', x * x) #st.sidebar.subheader('Choose the tokenizer', divider='grey') #option = st.sidebar.selectbox( # 'model_name', # ['deepseek-ai/deepseek-coder-1.3b-instruct', # 'bigcode/starcoder']) model_name_A = st.sidebar.text_input('Model Name A', 'deepseek-ai/deepseek-coder-1.3b-instruct') model_name_B = st.sidebar.text_input('Model Name B', 'deepseek-ai/deepseek-coder-1.3b-instruct') model_option = ['deepseek-ai/deepseek-coder-1.3b-instruct', 'MediaTek-Research/Breeze-7B-Instruct-64k-v0_1', 'microsoft/phi-2'] with st.sidebar.expander("Models that you might want"): for m in model_option: st.write(m) #'Your choice:', model_name st.sidebar.subheader('Write the input sentence', divider='grey') input_data = st.sidebar.text_input('Input Sentence', 'Hello sunshine!!!') col1, col2 = st.columns(2) with col1: st.subheader(model_name_A, divider='grey') res, token_num = get_res(model_name=model_name_A, input_sentence=input_data, single_print=False) st.subheader('Tokenized result') st.markdown(res, unsafe_allow_html=True) st.subheader('Number of tokens') st.markdown(f'{str(token_num)}', unsafe_allow_html=True) with col2: st.subheader(model_name_B, divider='grey') res, token_num = get_res(model_name=model_name_B, input_sentence=input_data, single_print=False) st.subheader('Tokenized result') st.markdown(res, unsafe_allow_html=True) st.subheader('Number of tokens') st.markdown(f'{str(token_num)}', unsafe_allow_html=True)