import pandas_profiling as pp import pandas as pd from datasets import load_dataset #LOINC datasetLOINC = load_dataset("awacke1/LOINC-CodeSet-Value-Description.csv") #SNOMED: datasetSNOMED = load_dataset("awacke1/SNOMED-CT-Code-Value-Semantic-Set.csv") #eCQM: dataseteCQM = load_dataset("awacke1/eCQM-Code-Value-Semantic-Set.csv") # map using autotokenizer from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") dataset = datasetLOINC.map(lambda examples: tokenizer(examples["Description"]), batched=True) JSONOBJ2=dataset['train'][0] sw = datasetLOINC.filter(lambda example: example["Description"].startswith("Allergy")) len(sw) print(sw) print(datasetLOINC) print(datasetSNOMED) print(dataseteCQM) # play with some dataset tools before the show: #print(start_with_ar["Description"]) #--- #Main Stage - Begin! #--- import os import json import numpy as np import gradio as gr HF_TOKEN = os.environ.get("HF_TOKEN") CHOICES = ["SNOMED", "LOINC", "CQM"] JSONOBJ = """{"items":{"item":[{"id": "0001","type": null,"is_good": false,"ppu": 0.55,"batters":{"batter":[{ "id": "1001", "type": "Regular" },{ "id": "1002", "type": "Chocolate" },{ "id": "1003", "type": "Blueberry" },{ "id": "1004", "type": "Devil's Food" }]},"topping":[{ "id": "5001", "type": "None" },{ "id": "5002", "type": "Glazed" },{ "id": "5005", "type": "Sugar" },{ "id": "5007", "type": "Powdered Sugar" },{ "id": "5006", "type": "Chocolate with Sprinkles" },{ "id": "5003", "type": "Chocolate" },{ "id": "5004", "type": "Maple" }]}]}}""" def profile_dataset(dataset=datasetSNOMED, username="awacke1", token=HF_TOKEN, dataset_name="awacke1/SNOMED-CT-Code-Value-Semantic-Set.csv"): df = pd.read_csv(dataset.Description) if len(df.columns) <= 15: profile = pp.ProfileReport(df, title=f"{dataset_name} Report") else: profile = pp.ProfileReport(df, title=f"{dataset_name} Report", minimal = True) repo_url = create_repo(f"{username}/{dataset_name}", repo_type = "space", token = token, space_sdk = "static", private=False) profile.to_file("./index.html") upload_file(path_or_fileobj ="./index.html", path_in_repo = "index.html", repo_id =f"{username}/{dataset_name}", repo_type = "space", token=token) readme = f"---\ntitle: {dataset_name}\nemoji: ✨\ncolorFrom: green\ncolorTo: red\nsdk: static\npinned: false\ntags:\n- dataset-report\n---" with open("README.md", "w+") as f: f.write(readme) upload_file(path_or_fileobj ="./README.md", path_in_repo = "README.md", repo_id =f"{username}/{dataset_name}", repo_type = "space", token=token) return f"Your dataset report will be ready at {repo_url}" #def lowercase_title(example): # return {"Description": example[title].lower()} # demonstrate map function of dataset #JSONOBJ_MAP=datasetLOINC.map(lowercase_title) #JSONOBJ_MAP=datasetLOINC.filter(lambda example: example["Description"].startswith("Mental health")) #def fn( text1, text2, num, slider1, slider2, single_checkbox, checkboxes, radio, dropdown, im1, im2, im3, im4, # video, audio1, audio2, file, df1, df2,): def fn( text1, text2, single_checkbox, checkboxes, radio, im4, file, df1, df2,): searchTerm = text1 searchTermSentence = text2 start_with_searchTermLOINC = datasetLOINC.filter(lambda example: example["Description"].startswith('Allergy')) #Allergy start_with_searchTermSNOMED = datasetSNOMED.filter(lambda example: example["Description"].startswith('Hospital')) #Hospital start_with_searchTermCQM = dataseteCQM.filter(lambda example: example["Description"].startswith('Telephone')) #Telephone #print(start_with_searchTermLOINC ) #print(start_with_searchTermSNOMED ) #print(start_with_searchTermCQM) #print(start_with_searchTermLOINC["train"][0] ) #print(start_with_searchTermSNOMED["train"][0] ) #print(start_with_searchTermCQM["train"][0] ) #returnMsg=profile_dataset() #print(returnMsg) # try: #top1matchLOINC = json.loads(start_with_searchTermLOINC['train']) #top1matchSNOMED = json.loads(start_with_searchTermSNOMED['train']) #top1matchCQM = json.loads(start_with_searchTermCQM['train']) # top1matchLOINC = json.loads(start_with_searchTermLOINC) # top1matchSNOMED = json.loads(start_with_searchTermSNOMED) # top1matchCQM = json.loads(start_with_searchTermCQM) # except: # print('Hello') #print(start_with_searchTermLOINC[0]) #print(start_with_searchTermSNOMED[0] ) #print(start_with_searchTermCQM[0] ) #print(returnMsg) # print("Datasets Processed") return ( #(text1 if single_checkbox else text2) + ", selected:" + ", ".join(checkboxes), # Text #(start_with_searchTermLOINC if single_checkbox else start_with_searchTermSNOMED) + ", selected:" + ", ".join(checkboxes), # Text # {"positive": num / (num + slider1 + slider2),"negative": slider1 / (num + slider1 + slider2),"neutral": slider2 / (num + slider1 + slider2),}, # Label # (audio1[0], np.flipud(audio1[1])) if audio1 is not None else os.path.join(os.path.dirname(__file__), "files/cantina.wav"), # Audio # np.flipud(im1) if im1 is not None else os.path.join(os.path.dirname(__file__), "files/cheetah1.jpg"), # Image # video if video is not None else os.path.join(os.path.dirname(__file__), "files/world.mp4"), # Video [ (JSONOBJ, "nn"), (JSONOBJ, "nn" ), (JSONOBJ, "nn" ), (searchTerm, "vrb"), ("The", "art"), ("quick brown", "adj"), ("fox", "nn"), ("jumped", "vrb"), ("testing testing testing", None), ("over", "prp"), ("the", "art"), ("testing", None), ("lazy", "adj"), ("dogs", "nn"), (".", "punc"), ] + [(f"test {x}", f"test {x}") for x in range(10)], # HighlightedText [ (JSONOBJ, 0.8 ), (JSONOBJ, 0.8 ), (JSONOBJ, 0.8 ), ("The testing testing testing", None), ("over", 0.6), ("the", 0.2), ("testing", None), ("lazy", -0.1), ("dogs", 0.4), (".", 0), ] + [(f"test", x / 10) for x in range(-10, 10)], # HighlightedText json.loads(JSONOBJ), # JSON #json.loads(JSONOBJ_MAP), # JSONOBJ_MAP #json.loads(top1matchLOINC), "", # HTML os.path.join(os.path.dirname(__file__), "files/titanic.csv"), df1, # Dataframe np.random.randint(0, 10, (4, 4)), # Dataframe df2, # Timeseries ) demo = gr.Interface( fn, inputs=[ gr.Textbox(value="Allergy", label="Textbox"), gr.Textbox(lines=3, value="Bathing", placeholder="Type here..", label="Textbox 2"), #gr.Number(label="Number", value=42), #gr.Slider(10, 20, value=15, label="Slider: 10 - 20"), #gr.Slider(maximum=20, step=0.04, label="Slider: step @ 0.04"), gr.Checkbox(label="Check for NER Match on Submit"), gr.CheckboxGroup(label="Clinical Terminology to Check", choices=CHOICES, value=CHOICES[0:2]), gr.Radio(label="Preferred Terminology Output", choices=CHOICES, value=CHOICES[2]), #gr.Dropdown(label="Dropdown", choices=CHOICES), #gr.Image(label="Image"), #gr.Image(label="Image w/ Cropper", tool="select"), #gr.Image(label="Sketchpad", source="canvas"), gr.Image(label="Webcam", source="webcam"), #gr.Video(label="Video"), #gr.Audio(label="Audio"), #gr.Audio(label="Microphone", source="microphone"), gr.File(label="File"), gr.Dataframe(label="Filters", headers=["Name", "Age", "Gender"]), gr.Timeseries(x="time", y=["price", "value"], colors=["pink", "purple"]), ], outputs=[ gr.Textbox(label="Textbox"), #gr.Label(label="Label"), #gr.Audio(label="Audio"), #gr.Image(label="Image"), #gr.Video(label="Video"), gr.HighlightedText(label="HighlightedText", color_map={"punc": "pink", "test 0": "blue"}), gr.HighlightedText(label="HighlightedText", show_legend=True), gr.JSON(label="JSON"), gr.HTML(label="HTML"), gr.File(label="File"), gr.Dataframe(label="Dataframe"), gr.Dataframe(label="Numpy"), gr.Timeseries(x="time", y=["price", "value"], label="Timeseries"), ], examples=[ [ "Allergy", "Admission", #10, #12, #4, True, ["SNOMED", "LOINC", "CQM"], "SNOMED", #"bar", #os.path.join(os.path.dirname(__file__), "files/cheetah1.jpg"), #os.path.join(os.path.dirname(__file__), "files/cheetah1.jpg"), #os.path.join(os.path.dirname(__file__), "files/cheetah1.jpg"), os.path.join(os.path.dirname(__file__), "files/cheetah1.jpg"), #os.path.join(os.path.dirname(__file__), "files/world.mp4"), #os.path.join(os.path.dirname(__file__), "files/cantina.wav"), #os.path.join(os.path.dirname(__file__), "files/cantina.wav"), os.path.join(os.path.dirname(__file__), "files/titanic.csv"), [[1, 2, 3], [3, 4, 5]], os.path.join(os.path.dirname(__file__), "files/time.csv"), ] ] * 3, theme="default", title="⚗️🧠🔬🧬 Clinical Terminology Auto Mapper AI 👩‍⚕️🩺⚕️🙋", cache_examples=False, description="Clinical Terminology Auto Mapper AI", article="Learn more at [Yggdrasil](https://github.com/AaronCWacker/Yggdrasil)", # live=True, ) if __name__ == "__main__": demo.launch(debug=True)