Update app.py

app.py

@@ -1,6 +1,7 @@
 import pandas_profiling as pp
 import pandas as pd
 from datasets import load_dataset
+
 #LOINC
 datasetLOINC = load_dataset("awacke1/LOINC-CodeSet-Value-Description.csv")
 #SNOMED:
@@ -8,6 +9,11 @@ datasetSNOMED = load_dataset("awacke1/SNOMED-CT-Code-Value-Semantic-Set.csv")
 #eCQM:
 dataseteCQM = load_dataset("awacke1/eCQM-Code-Value-Semantic-Set.csv")
 
+# map using autotokenizer
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+dataset = dataset.map(lambda datasetLOINC : tokenizer(datasetLOINC["Description"]), batched=True)
+
 print(datasetLOINC)
 print(datasetSNOMED)
 print(dataseteCQM)
@@ -78,15 +84,19 @@ def fn(    text1,    text2,    single_checkbox,    checkboxes,    radio,    im4,
     #print(returnMsg)
     
     try:
-        top1matchLOINC = json.loads(start_with_searchTermLOINC['train'])
-        top1matchSNOMED = json.loads(start_with_searchTermSNOMED['train'])
-        top1matchCQM = json.loads(start_with_searchTermCQM['train'])
+        #top1matchLOINC = json.loads(start_with_searchTermLOINC['train'])
+        #top1matchSNOMED = json.loads(start_with_searchTermSNOMED['train'])
+        #top1matchCQM = json.loads(start_with_searchTermCQM['train'])
+        top1matchLOINC = json.loads(start_with_searchTermLOINC)
+        top1matchSNOMED = json.loads(start_with_searchTermSNOMED)
+        top1matchCQM = json.loads(start_with_searchTermCQM)
     except:
-        print(start_with_searchTermLOINC[0])
-        print(start_with_searchTermSNOMED[0] )
-        print(start_with_searchTermCQM[0] )
+        print('Hello')
+        #print(start_with_searchTermLOINC[0])
+        #print(start_with_searchTermSNOMED[0] )
+        #print(start_with_searchTermCQM[0] )
         
-    print(returnMsg)
+    #print(returnMsg)
     print("Datasets Processed")
     
     return (