jamino30 commited on
Commit
9e2a2bb
1 Parent(s): 0097326

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +3 -1
app.py CHANGED
@@ -2,7 +2,9 @@ import gradio as gr
2
  from xsbpe.basic import BasicTokenizer
3
 
4
  tk = BasicTokenizer()
5
- tk.train(open('dune.txt').read(), 256 + 10000, verbose=False)
 
 
6
 
7
  def tokenize(text):
8
  tokens = tk.encode(text)
 
2
  from xsbpe.basic import BasicTokenizer
3
 
4
  tk = BasicTokenizer()
5
+ print('Tokenizer initialized.')
6
+ tk.train(open('dune.txt').read(), 256 + 10, verbose=False)
7
+ print('Training complete.')
8
 
9
  def tokenize(text):
10
  tokens = tk.encode(text)