Upload 5 files

README.md

@@ -1,13 +1,12 @@
 ---
-title: Qa App Overlap
-emoji: 🏢
+title: Overlap Qa Check
+emoji: 🌍
 colorFrom: yellow
-colorTo: blue
+colorTo: green
 sdk: gradio
-sdk_version: 4.1.2
+sdk_version: 3.29.0
 app_file: app.py
 pinned: false
-license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,159 @@
+from datetime import datetime
+
+import gradio as gr
+import gspread
+import validators
+from validators import ValidationFailure
+from google.oauth2.service_account import Credentials
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive 
+
+import etl
+
+# test sheet - https://docs.google.com/spreadsheets/d/1iJ0-882HsWkAth0e0P_kf21aL6583Z7m1LwloaRCVEc/edit#gid=0
+week_list = [f"week_{i}" for i in range(1, 51)]
+
+# credentials file 
+cred_file_path = 'heliumhealth-a05d595e5991.json'
+
+# google auth scopes
+scopes = ['https://www.googleapis.com/auth/spreadsheets',
+          'https://www.googleapis.com/auth/drive']
+
+# create credentials 
+credentials = Credentials.from_service_account_file(
+    cred_file_path, scopes=scopes )
+
+# authorize google spreadsheet
+gc = gspread.authorize(credentials)
+
+gauth = GoogleAuth()
+drive = GoogleDrive(gauth)
+
+def overlap_matching(row, d_type):
+  """
+  matching overlap 
+  """
+  match_cols = {
+      "prescription": ['RX Norm [Super Generic]_x', 'RX Norm [Super Generic]_y'],
+      "diagnosis": ['ICD10 Diagnosis_x','ICD10 Diagnosis_y']
+  }
+  cols = match_cols[d_type]
+  row['match_status'] = 'match' if row[cols[0]] == row[cols[1]] else 'no match'
+  return row
+
+def overlap_check(gs, start_date, end_date, week_no, d_type):
+    """
+   load, preprocess, check overlap, postprocess and output data to google sheet 
+
+    Args:
+        gs (GSheet instance): Gsheet instance access to google sheet 
+        start_date (str): Date str format(YYYY-mm-dd)
+        end_date (_type_): Date str format(YYYY-mm-dd)
+        d_type (_type_):  sheet type (prescription or diagnosis)
+    """
+    # load data
+    all_data = etl.load_data(gs, start_date, end_date, d_type)
+
+    # preprocess and return overlap data
+    overlap_data = etl.preprocess_data(all_data, d_type)
+
+    # do overlap matching 
+    overlap_data = overlap_data.apply(overlap_matching, axis=1, args=[d_type])
+
+    # post process 
+    overlap_data = etl.post_process(overlap_data, d_type)
+
+    # write to sheet 
+    etl.output_data(gc, overlap_data, week_no, d_type)
+
+
+def overlap_check_main(sheet_type, start_date_str, end_date_str, week_str, sheet_url):
+    """
+    overlap check main function
+
+    Args:
+        sheet_type (str): sheet type (prescription or diagnosis)
+        start_date_str (str): start date string: e.g. 2023-03-21 
+        end_date_str (str): end date string: e.g. 2023-03-24
+        week_str (str): week string e.g. week_1
+        sheet_url (url): _description_
+
+    Raises:
+        gr.exceptions.Error: Date Format Error - either start_date or end date are bad format
+        gr.exceptions.Error: Date Error - when start_date is greater than end date
+        gr.exceptions.Error: URL Error - Bad url format
+    """
+    # format date from string
+    try: 
+        start_date = datetime.strptime(start_date_str.strip(), "%Y-%m-%d").date()
+        end_date = datetime.strptime(end_date_str.strip(), "%Y-%m-%d").date()
+    except:
+        raise gr.Error(message="Wrong date format")
+
+    # raise error when start date is greater end date
+    if start_date >= end_date:
+        raise gr.Error(message="Start date cannot be greater end date")
+    
+    # Check if the input is valid url
+    # ToDO: Error message displayed is not explanatory - Fix it 
+    url_check = validators.url(sheet_url)
+    if isinstance(url_check, ValidationFailure):
+        raise gr.Error(message="Please enter a valid URL")
+    
+    
+    # open the google sheet for reading 
+    
+    gs = gc.open_by_url(sheet_url)
+    
+
+    # if sheet_type == 'prescription':
+    try:
+        overlap_check(gs, start_date_str, end_date_str, week_str,  sheet_type)
+    except:
+        gr.Error(message="Permission denied. Please add IAM user to the sheet and try again")
+
+
+    return f"Successfully ran {sheet_type} overlap check for {week_str.replace('_', ' ').title()} ({start_date_str} - {end_date_str})"
+
+with gr.Blocks() as demo:
+  gr.Markdown(
+      """
+      ## Overlap Check App
+
+      * Add IAM User to sheet you want to test
+      * Gsheet tabs required for diagnosis:  
+        * Diagnosis
+      * Gsheet tabs required for prescription:  
+        * Prescriptions
+      * Data headers required for diagnosis
+        * Unstructured Name, ICD10 Diagnosis, Intern
+      * Data headers required for prescription
+        * Unstructured Name, RX Norm [Super Generic], Intern 
+      """
+  )
+  # inputs
+  sheet_type = gr.Dropdown(['prescription', 'diagnosis'], label="QA Type")
+  start_date = gr.Textbox(label="Start Date", placeholder="YYYY-MM-DD")
+  end_date = gr.Textbox(label="End Date", placeholder="YYYY-MM-DD")
+  week_input = gr.Dropdown(week_list, label="Week")
+  url = gr.Textbox(label="URL", placeholder="Enter sheet url ...")
+  
+  # outputs
+  output = gr.Textbox(label="Output Box")
+  
+  run_btn = gr.Button("Run")
+  run_btn.click(
+      fn=overlap_check_main,
+      inputs=[
+          sheet_type,
+          start_date,
+          end_date, 
+          week_input, 
+          url
+          ],
+      outputs=output, 
+      api_name="Overlap_check"
+  )
+
+demo.launch()

etl.py

@@ -0,0 +1,159 @@
+import gradio as gr 
+import pandas as pd
+from gspread_dataframe import set_with_dataframe  
+
+def load_data(gs, start_date, end_date, d_type):
+    """
+    load data from google sheet
+
+    Args:
+        gs (Gsheet instance ): GSheet instance with access to google sheet
+        start_date (str): Date str format(YYYY-mm-dd)
+        end_date (str): Date str format(YYYY-mm-dd)
+        d_type (str):sheet type (prescription or diagnosis)
+
+    Raises:
+        gr.Error: 403 permission denied
+
+    Returns:
+        data (DataFrame): loaded data from google sheet
+    """
+    sheets_dict = {
+        "diagnosis": "Diagnosis",
+        "prescription": "Prescriptions"
+    }
+
+    # open data and super sheet
+    try:
+        data_sheet = gs.worksheet(sheets_dict[d_type])
+    except:
+        raise gr.Error(message='Permission denied. Please add IAM user to the sheet and try again')
+    
+    # read data from sheets
+    all_data = pd.DataFrame(data_sheet.get_all_records())
+    all_data.columns = all_data.columns.str.strip()
+
+    # transform date fields
+    all_data['date_cleaned'] = pd.to_datetime(all_data['date_cleaned'], infer_datetime_format=True)
+    all_data['date_reviewed'] = pd.to_datetime(all_data['date_reviewed'], infer_datetime_format=True)
+
+    data = all_data[(all_data['date_cleaned'] >= start_date) & (all_data['date_cleaned'] <= end_date)]
+
+    return data
+
+def preprocess_data(data, d_type):
+    """
+    preprocess loaded data from google sheet
+
+    Args:
+        data (DataFrame):  google sheet data as a dataframe
+        d_type (Str): sheet type (prescription or diagnosis)
+
+    Raises:
+        gr.Error (ValueError): Wrong column
+
+    Returns:
+        Overlap (DataFrame): Overlap data
+    """
+
+    data.columns = data.columns.str.strip()
+
+    overlap_cols = {
+       "prescription": ["RX Norm [Super Generic]_x", "RX Norm [Super Generic]_y" ], 
+       "diagnosis": ["ICD10 Diagnosis_x", "ICD10 Diagnosis_y"]
+    }
+
+    try:
+        # extract the interns
+        intern_1 = data['Intern'].unique()[0]
+        intern_2 = data['Intern'].unique()[1]
+
+        # intern 1 and intern 2 data 
+        intern_1_data = data[data['Intern'] == intern_1.strip()]
+        intern_2_data = data[data['Intern'] == intern_2.strip()]
+
+        # extract the overlap
+        overlap = pd.merge(intern_1_data, intern_2_data, on='Unstructured Name')
+
+        # remove all white spaces 
+        col = overlap_cols[d_type]
+        overlap[col[0]] = overlap[col[0]].str.strip()
+        overlap[col[1]] = overlap[col[1]].str.strip()
+    except:
+       raise gr.Error(message="No Intern Column")
+
+    return overlap
+
+def post_process(data, d_type):
+    """
+   post process data
+
+    Args:
+        data (DataFrame): matched data as dataframe
+        d_type (str): sheet type (prescription or diagnosis)
+
+    Returns:
+        data (DataFrame): postprocessed data as dataframe
+    """
+    drop_cols = {
+        "prescription": ['Type_y','Supervisor_y', 'date_cleaned_y', 'date_reviewed_y'], 
+        "diagnosis": ['Supervisor_y', 'date_cleaned_y', 'date_reviewed_y']
+    }
+    post_cols = {
+        "prescription": ['Unstructured Name', 'Intern_x', 'Status_x', 'RX Norm [Super Generic]_x', 
+                            'Intern_y', 'Status_y', 'RX Norm [Super Generic]_y',  'match_status', 'Type_x', 
+                            'Supervisor_x', 'date_cleaned_x', 'date_reviewed_x'],
+        "diagnosis":   ['Unstructured Name', 'Intern_x', 'Status_x', 'ICD10 Diagnosis_x', 
+                        'Intern_y', 'Status_y', 'ICD10 Diagnosis_y',  'match_status',
+                        'Supervisor_x', 'date_cleaned_x', 'date_reviewed_x']
+        }
+    
+    d_cols = drop_cols[d_type]
+    p_cols = post_cols[d_type]
+
+    data = data.drop(columns=d_cols)
+    data = data[p_cols]
+
+    data = data.rename(columns={
+        'date_cleaned_x': 'date_cleaned',
+        'date_reviewed_x': 'date_reviewed'
+        })
+    
+    return data
+
+def output_data(gc, data, week_no, dtype):
+    """
+    outputs data to a google sheet
+
+    Args:
+        gc (GSheet instance): Gsheet permission instance
+        data (DataFrame):  Data to write to google sheet
+        week_no (str): Week no e.g. week_1
+        d_type (str): sheet type (prescription or diagnosis)
+    
+    Raises:
+        gr.exception.Error: Sheet Error
+    """
+    out_sheet_dict = {
+        "diagnosis": "https://docs.google.com/spreadsheets/d/1UJ2PHR62mcz11D2qeX-Wk2fs1357BW78o8g76lwG93w/edit#gid=2123373592", 
+        "prescription": "https://docs.google.com/spreadsheets/d/1Fo9V6J_L9eWX3qEScITP2L8nI9exk_VslGMgk-1jCJw/edit#gid=1287467149"
+    }
+
+    outsheet_url = out_sheet_dict[dtype]
+    out_gs = gc.open_by_url(outsheet_url)
+
+    # write output to sheet 
+    try:
+        out_worksheet = out_gs.worksheet(week_no)
+        # clear the worksheet 
+        out_worksheet.clear()
+    except:
+        raise gr.Error(f"Result sheet for {week_no} does not exist")
+
+    # write dataframe to work sheet
+    set_with_dataframe(
+        worksheet=out_worksheet, 
+        dataframe=data, 
+        include_index=False,
+        include_column_header=True, 
+        resize=True)

heliumhealth-a05d595e5991.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "heliumhealth",
+  "private_key_id": "a05d595e59911b601cf718b32e967a0f4e1f463d",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCzJJ2dCd32LwAc\njoTiCB2shYwGRFvApmF3/RUkT9BZDg5S+imfuXSsHBgueKIEAW7E0UBGVbioKQC0\nEzKh/+aLVdZEYdqyNdrx4g5HmuDn4sQyd/zP0J3cUa/LlF/cND6HsExJcdkjL9rV\nVZEQZhpVrgB9J6Ui5dZsrN9RWHguP8awI6JFqsfF2aKWJ4Z+Gl9YlxMhCqcwBalF\nf4MUOwnXSK1t0QMXdEAg9zA+wwySYjCz/nVOIrEBB1lHp1mEiwpv2KtuX3f2GGRt\nSGwOThDWJPn6EuLcX2mlxEs7ylEs1hYcKfROPzBMSRX5g/m5lKkRuZLsBpqKZj5G\nrB62FasXAgMBAAECggEADIMutRTG2Zaf0d7MfkMl23J6fMeShwq3/RsCuoYOneHC\n813AwYTBFvJhaAGKA3CcLYgWDrOXEbvHbkx2+WS31BybOiyBi3gCgmmRKMQzGE84\n9/N0h4lhNU3JUxVaerV1dqHEOk8G5RPHG6Z92uDezwQ3yYYW3TCjgPS0JmjhAA3b\nAnAgATuiY4ivscR28xSTDsexQhDnVGdM0ms4hrAUV0nM+sGLbXil6hTBoRi/iEmq\nGESeFsDOcdJKhIAvs94FwhHGLljCniP+2/jN6Wg8+wTFwUpaH0NpV6tfHYvGe8Bp\natfVZwCqwP5+J3dXHuaNUwe/cUK6HSVcGARXp5vubQKBgQD7xXVg2WoMONpGHWnD\nxk3T085JiC+MgBK+y5MzNLTtVxCNzsv3I70lu8ilv7ytfy35bnQ2dx8Ych1Jjc7z\n5kQs9rRxtY5iZvyTRaBB4Zfyr2uGxrR/lMJizsH+M0my48zPiHWLdqJSTupBmufr\n2kFeRIbphEl6u5fir9s5c/+EGwKBgQC2JuCQXrYHryn2vs6qf+Zf/RgzK5+3irHy\nEeYbedEpe6dCy5dWjlh7sXR23938XTXtnwmKWMuovdnLybBXvuWacOsqz5wYuXAn\nK6V8oBPpV9Aj+nVSft26E8nMzNX1lXbvxBgq1GvOUk7Mbi9p4nm6gjKNKYo1/U9P\nH+S9VHQMtQKBgAxqkH9WOYSgySLsEGs3PF9V8rZtoOqs5j/Cil/cGZAa2xYjPKvT\nd2CFAkAqVIO54eqLJ/AHr+Dkv80A0VP15ybQg9WXvo3bxlj89gpJtdSiEgtzgTNJ\ngsycpbSDkv7ffRo/AI0ALMEiYysZGJbpDJA5kO2zOGx1E+h6A7WxoshBAn8UolLO\nB+yW8kDOss62gcaXGRSkt0xgflWqFlz6v9Hx4RARgP6jz3w2huOqk7GR5P027c0m\n3ugzNU52x2Iyjm10EVaSgvIr2tXZmhglBf07cbciXDzuG3ECozs49/tE1qmif5Q9\nRdLwjGJgxhqY5A7mEdmoJAyEES3qyLIgwHBlAoGAS0sA8A5Ay+BbaAGufqSaBnxt\nQgMGNYUgyeG6edksXMRPMqTWbQ5xdjEmv38nQ8AT9RVMZ9TfZqCts1fCD+FgTcAU\niAasrJMvd4mCBhWZWPwGJN+yaeZHvdbrXsSEo3kkyiwjFip03CCJtDJO3MwyowOb\nIxC34Nn4A6XUGjaF7Io=\n-----END PRIVATE KEY-----\n",
+  "client_email": "test-gs@heliumhealth.iam.gserviceaccount.com",
+  "client_id": "102772078802762265627",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-gs%40heliumhealth.iam.gserviceaccount.com"
+}

requirements.txt

@@ -0,0 +1,88 @@
+aiofiles==23.1.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==4.2.2
+anyio==3.6.2
+async-timeout==4.0.2
+attrs==23.1.0
+beautifulsoup4==4.12.2
+cachetools==5.3.0
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.3
+contourpy==1.0.7
+cycler==0.11.0
+decorator==5.1.1
+entrypoints==0.4
+fastapi==0.95.1
+ffmpy==0.3.0
+filelock==3.12.0
+fonttools==4.39.3
+frozenlist==1.3.3
+fsspec==2023.5.0
+google==3.0.0
+google-api-core==2.11.0
+google-api-python-client==2.86.0
+google-auth==2.17.3
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==1.0.0
+googleapis-common-protos==1.59.0
+gradio==3.28.3
+gradio_client==0.2.0
+gspread==5.8.0
+gspread-dataframe==3.3.0
+h11==0.14.0
+httpcore==0.17.0
+httplib2==0.22.0
+httpx==0.24.0
+huggingface-hub==0.14.1
+idna==3.4
+Jinja2==3.1.2
+jsonschema==4.17.3
+kiwisolver==1.4.4
+linkify-it-py==2.0.2
+markdown-it-py==2.2.0
+MarkupSafe==2.1.2
+matplotlib==3.7.1
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+multidict==6.0.4
+numpy==1.24.3
+oauth2client==4.1.3
+oauthlib==3.2.2
+orjson==3.8.12
+packaging==23.1
+pandas==2.0.1
+Pillow==9.5.0
+protobuf==4.22.4
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==1.10.7
+PyDrive==1.3.1
+pydub==0.25.1
+Pygments==2.15.1
+pyparsing==3.0.9
+pyrsistent==0.19.3
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3
+PyYAML==6.0
+requests==2.30.0
+requests-oauthlib==1.3.1
+rsa==4.9
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.4.1
+starlette==0.26.1
+toolz==0.12.0
+tqdm==4.65.0
+typing_extensions==4.5.0
+tzdata==2023.3
+uc-micro-py==1.0.2
+uritemplate==4.1.1
+urllib3==2.0.2
+uvicorn==0.22.0
+validators==0.20.0
+websockets==11.0.3
+yarl==1.9.2