Update game1.py

game1.py

@@ -5,6 +5,9 @@ import pandas as pd
 import gradio as gr
 import numpy as np
 import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers import pipeline
+import shap
 
 def read1(lang, num_selected_former):
     if lang in ['en']:
@@ -107,13 +110,11 @@ def func1(lang_selected, num_selected, human_predict, num1, num2, user_important
 
     # (START) off-the-shelf version -- slow at the beginning
     # Load model directly
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    
+
     tokenizer = AutoTokenizer.from_pretrained("nlptown/bert-base-multilingual-uncased-sentiment")
     model = AutoModelForSequenceClassification.from_pretrained("nlptown/bert-base-multilingual-uncased-sentiment")
 
     # Use a pipeline as a high-level helper
-    from transformers import pipeline
 
     device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
     print(device)
@@ -272,13 +273,21 @@ def func1(lang_selected, num_selected, human_predict, num1, num2, user_important
 def interpre1(lang_selected, num_selected):
     if lang_selected in ['en']:
         fname = 'data1_en.txt'
+        tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
     else:
         fname = 'data1_nl_10.txt'
+        tokenizer = AutoTokenizer.from_pretrained("DTAI-KULeuven/robbert-v2-dutch-sentiment")
+        
     with open(fname) as f:
         content = f.readlines()
         text = eval(content[int(num_selected*2)])
         interpretation = eval(content[int(num_selected*2+1)])
-    
+
+    encodings = tokenizer(text['text'], is_pretokenized=False, return_offsets_mapping=True)
+    print(encodings['offset_mapping'])
+    is_subword = np.array(encodings['offset_mapping'])[:,0] != 0
+    print(is_subword)
+    print(abc)
     print(interpretation)
 
     res = {"original": text['text'], "interpretation": interpretation}
@@ -337,8 +346,6 @@ def func1_written(text_written, human_predict, lang_written):
     ''' 
 
     # (START) off-the-shelf version
-    from transformers import AutoTokenizer, AutoModelForSequenceClassification
-    from transformers import pipeline
 
 
     # tokenizer = AutoTokenizer.from_pretrained("nlptown/bert-base-multilingual-uncased-sentiment")
@@ -373,9 +380,6 @@ def func1_written(text_written, human_predict, lang_written):
             ai_predict += int(random.randint(-1, 1))
         chatbot.append(("AI thinks in a different way from human. 😉", "⬅️ Feel free to try another one! ⬅️"))
 
-
-    import shap
-
     # sentiment_classifier = pipeline("text-classification", return_all_scores=True)
     device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")