Added token IDs

SmilesPeTokenizer.py

@@ -1,12 +1,16 @@
-import codecs
-from SmilesPE.tokenizer import *
+from tokenizers import Tokenizer
 
-def smilespe_tokenizer(smiles_string):
+def bpe_tokenizer(smiles_string):
+    # Load the tokenizer from the saved file
+    tokenizer = Tokenizer.from_file("chembl_bpe_tokenizer.json")
 
-    spe_vob = codecs.open('chembl_smiles_tokenizer30000.txt')
-    spe = SPE_Tokenizer(spe_vob)
+    # Tokenize the SMILES string
+    encoded_output = tokenizer.encode(smiles_string)
 
-    tokenized = spe.tokenize(smiles_string)
-
-    return tokenized
+    # To get the tokenized output as text
+    tokens_text = encoded_output.tokens
 
+    # To get the corresponding token IDs
+    token_ids = encoded_output.ids
+    
+    return tokens_text, token_ids

app.py

@@ -6,7 +6,7 @@ iface = gr.Interface(
     inputs=[
         gr.Textbox(label="SMILES"),         
     ],
-    outputs="text"
+    outputs=["text", "text"]
 )
 
 iface.launch()