add process function

app.py

@@ -1,9 +1,10 @@
 import streamlit as st
+from utils import get_res
 
-st.title('Tokenizers demo!!')
+st.title('Tokenizers demo')
 
-x = st.slider('Select a value')
-st.write(x, 'squared is', x * x)
+#x = st.slider('Select a value')
+#st.write(x, 'squared is', x * x)
 
 
 option = st.selectbox(
@@ -14,4 +15,11 @@ option = st.selectbox(
 
 model_name = st.text_input('Model Name', 'deepseek-ai/deepseek-coder-1.3b-instruct')
 
-'Your choice:', model_name
+'Your choice:', model_name
+
+input_data = st.text_input('Input Sentence', 'Hello world!!!')
+
+
+res = get_res(model_name=model_name, input_sentence=input_data, single_print=False)
+
+st.write(res)

utils.py

@@ -0,0 +1,24 @@
+
+from transformers import AutoTokenizer
+import itertools
+
+
+
+def get_color():
+    colors = [i for i in range(41, 48)]
+    return itertools.cycle(colors)
+
+def get_res(model_name, input_sentence, single_print=True):
+    
+    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+    color_iterator = get_color()
+    out = tokenizer.encode(input_sentence, add_special_tokens=False)
+    token_num = len(out)
+
+    w = [ '\033[''1;'+str(next(color_iterator))+f'm {tokenizer.decode(x)}\033[m' for x in out]
+    res = ''.join(w) + f' {str(token_num)}'
+    if single_print:
+        print(res)
+    else:
+        return res
+