Update app.py

app.py

@@ -66,6 +66,30 @@ def profile_dataset(dataset=datasetSNOMED, username="awacke1", token=HF_TOKEN, d
 #JSONOBJ_MAP=datasetLOINC.map(lowercase_title)
 #JSONOBJ_MAP=datasetLOINC.filter(lambda example: example["Description"].startswith("Mental health")) 
 
+
+
+
+def concatenate_text(examples):
+    return {
+        "text": examples["Code"]
+        + " \n "
+        + examples["Description"]
+        + " \n "
+        + examples["Purpose: Clinical Focus"]
+    }
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:, 0]
+    
+def get_embeddings(text_list):
+    encoded_input = tokenizer(
+        text_list, padding=True, truncation=True, return_tensors="tf"
+    )
+    encoded_input = {k: v for k, v in encoded_input.items()}
+    model_output = model(**encoded_input)
+    return cls_pooling(model_output)
+
+
 def fn(    text1,    text2,    num,    slider1,    slider2,    single_checkbox,    checkboxes,    radio,    dropdown,    im1,    im2,    im3,    im4,
     video,    audio1,    audio2,    file,    df1,    df2,):
 #def fn(    text1,    text2,    single_checkbox,    checkboxes,    radio,    im4,  file,    df1,    df2,):
@@ -73,7 +97,10 @@ def fn(    text1,    text2,    num,    slider1,    slider2,    single_checkbox,
     searchTerm = text1
     searchTermSentence = text2
     
-    start_with_searchTermLOINC = datasetLOINC.filter(lambda example: example["Description"].startswith('Allergy'))    #Allergy
+    start_with_searchTermLOINC = datasetLOINC.filter(lambda example:example["Description"].startswith('Allergy'))    #Allergy
+    
+    
+    # FAISS
     columns = start_with_searchTermLOINC.column_names
     columns_to_keep = ["Value Set Name", "Code", "Description", "Purpose: Clinical Focus", "Code System OID"]
     columns_to_remove = set(columns_to_keep).symmetric_difference(columns)
@@ -81,11 +108,68 @@ def fn(    text1,    text2,    num,    slider1,    slider2,    single_checkbox,
     start_with_searchTermLOINC 
     start_with_searchTermLOINC.set_format("pandas")
     df = start_with_searchTermLOINC[:]
-    #df["Purpose: Clinical Focus"][0].tolist()
+
     df["Purpose: Clinical Focus"][0]
+    
     df4 = df.explode("Purpose: Clinical Focus", ignore_index=True)
     df4.head(4)
+    
+    from datasets import Dataset
+    clinical_dataset = Dataset.from_pandas(df4)
+    clinical_dataset
+    
+    clinical_dataset = clinical_dataset.map(lambda x: {"c_length": len(x["Description"].split())})
+    
+    clinical_dataset = clinical_dataset.filter(lambda x: x["c_length"] > 15)
+    clinical_dataset
+    
+    
+    clinical_dataset = clinical_dataset.map(concatenate_text)
+    embedding = get_embeddings(comments_dataset["text"][0])
+    embedding.shape
+    
+    from transformers import AutoTokenizer, TFAutoModel
+
+    model_ckpt = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+    tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
+    model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)
+    
+    TensorShape([1, 768])
+    
+    embeddings_dataset = comments_dataset.map(
+    lambda x: {"embeddings": get_embeddings(x["text"]).numpy()[0]})
+    
+    embeddings_dataset.add_faiss_index(column="embeddings")
 
+    question = "How can I load a dataset offline?"
+    question_embedding = get_embeddings([question]).numpy()
+    question_embedding.shape
+    
+    scores, samples = embeddings_dataset.get_nearest_examples("embeddings", question_embedding, k=5)
+    
+    import pandas as pd
+
+    samples_df = pd.DataFrame.from_dict(samples)
+    samples_df["scores"] = scores
+    samples_df.sort_values("scores", ascending=False, inplace=True)
+    
+    
+    #        "text": examples["Code"]
+    #    + " \n "
+    #    + examples["Description"]
+    #    + " \n "
+    #    + examples["Purpose: Clinical Focus"]
+        
+        
+    for _, row in samples_df.iterrows():
+        print(f"Code: {row.Code}")
+        print(f"Description: {row.Description}")
+        #print(f"Purpose: Clinical Focus: {row.Purpose: Clinical Focus}")
+        #print(f"URL: {row.html_url}")
+        print("=" * 50)
+        print()
+    
+    # SNOMED and CQM ---------------
     start_with_searchTermSNOMED = datasetSNOMED.filter(lambda example: example["Description"].startswith('Hospital'))    #Hospital
     start_with_searchTermCQM = dataseteCQM.filter(lambda example: example["Description"].startswith('Telephone'))    #Telephone