merged github/demo_huggingFace into runner/main

bin/models/claims_kmn_py3816_sk111hp_22cols.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a3f3e93e08cc64a169e199232261b55250ca7c6599522cea2c2821d99edb554
+size 2234618

bin/models/claims_stdScl_py3816_sk111hp_27cols.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84d5379c031897e5df1ecbc6b07dca005873955818285f98cb1c8ec5d291c581
+size 1779

bin/models/prov_gbc_py3816_sk111hp_32cols.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a740ba487ec674f9eafdc68f360e98be7b4b834fac0a6a79f9b82bac583d710f
+size 45135

bin/models/prov_stdScl_py3816_sk111hp_32cols.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2137e7adba591cb760d4b73c561f84da5cc2aa49235000b274304f37d98582b7
+size 2094

data/demo_data/20230210165948_provTestSample.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb380a5e4cfed980e8514bcee519f45de67556ffdd09e3eaf9d1f635c1c77d79
+size 7419701

data/demo_data/20230210170628_claimsTestSample.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5b35e7014ab77ba73140f875637903691bb4f22019d2b956a320b5d0b5c8aa2
+size 6418423

data/test_claims.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478c2854368471f7db414171bb5c1fc8904fbea49fb5aa3091a58f41443f8bed
+size 61997959

lib/claims.py

@@ -0,0 +1,258 @@
+import pandas as pd
+import lib.utils as libPaths
+
+from lib.models import mdl_utils, mdl_xgb, mdl_logR, mdl_svm
+from lib.models import mdl_autoenc, mdl_kmeans
+
+
+m_blnTraceOn = True
+m_blnTrace2On = False
+
+#--- load, merge data from file
+m_kstrDataPath = libPaths.pth_data
+m_kstrModelPath = libPaths.pth_model
+m_kstrBinModelPath = libPaths.pth_binModels
+
+
+
+def getPath_defPklClaims(blnIsTrain=False):
+    global m_kstrDataPath
+    strPrefix="test_"
+    if (blnIsTrain):  strPrefix = "train_"
+    strPth_pklClaims = m_kstrDataPath + strPrefix + 'claims.pkl'
+    return strPth_pklClaims
+
+
+
+def load_claims(blnIsTrain=False, blnForceCsv=False):
+    if (blnForceCsv):  
+        pdfClaims = loadCsv_claims(blnIsTrain)
+    else:
+        pdfClaims = loadPkl_claims(blnIsTrain)
+    return pdfClaims
+
+
+
+def loadCsv_claims(blnIsTrain=False):
+    global m_kstrDataPath
+    #--- load all csv test data
+    if (blnIsTrain):
+        print("INFO (loadCsv_claimsData):  load train data ...")
+        strPthProvider = m_kstrDataPath + 'Train-1542865627584.csv'
+        strPthBenef = m_kstrDataPath + 'Train_Beneficiarydata-1542865627584.csv'
+        strPthInpat = m_kstrDataPath + 'Train_Inpatientdata-1542865627584.csv'
+        strPthOutpat = m_kstrDataPath + 'Train_Outpatientdata-1542865627584.csv'
+    else:
+        print("INFO (loadCsv_claimsData):  load test data ...")
+        strPthProvider = m_kstrDataPath + 'Test-1542969243754.csv'
+        strPthBenef = m_kstrDataPath + 'Test_Beneficiarydata-1542969243754.csv'
+        strPthInpat = m_kstrDataPath + 'Test_Inpatientdata-1542969243754.csv'
+        strPthOutpat = m_kstrDataPath + 'Test_Outpatientdata-1542969243754.csv'
+    
+    #--- output:  pandas data frame
+    pdfProvider = pd.read_csv(strPthProvider) 
+    pdfBenef = pd.read_csv(strPthBenef)
+    pdfInpat = pd.read_csv(strPthInpat)
+    pdfOutpat = pd.read_csv(strPthOutpat) 
+
+    #--- data engineering 
+    pdfBenef = prep_benefData(pdfBenef)
+    pdfInpat = prep_inpatData(pdfInpat)
+
+    #--- merge inpatient and outpatient data (assert: 31 cols)
+    aryMergeCols = list(pdfOutpat.columns)
+    pdfAllpat = pdfInpat.merge(pdfOutpat, on=aryMergeCols, how='outer')
+
+    #--- +merge beneficiary data
+    pdfAllPatBenef = pdfAllpat.merge(pdfBenef, on='BeneID', how='inner')
+
+    #--- +merge provider data
+    pdfAllPatBenefProv = pdfAllPatBenef.merge(pdfProvider, on='Provider', how='inner')
+    
+    #--- export data
+    strPth_pklClaims = getPath_defPklClaims(blnIsTrain)
+    print("TRACE (claims.loadCsv_claims):  pkl claim data file path ... ", strPth_pklClaims)
+    pdfAllPatBenefProv.to_pickle(strPth_pklClaims)
+
+    #print("INFO (csvClaims.shape):  ", pdfTest_allPatBenefProv.shape)
+    return pdfAllPatBenefProv
+
+
+
+def loadCsv_testClaims():
+    #--- TODO:  make optional arg test or train data
+    return loadCsv_claims(False)
+
+
+
+def loadPkl_claims(blnIsTrain=False):
+    strPth_pklClaims = getPath_defPklClaims(blnIsTrain)
+    try:
+        pdfClaims = pd.read_pickle(strPth_pklClaims)
+    except FileNotFoundError:
+        #--- catch:  there is no pickle file
+        #--- load from csv instead;  will create pkl files for next time
+        pdfClaims = loadCsv_claims(blnIsTrain)
+    return pdfClaims
+
+
+
+#--- feat eng
+def do_featEng(pdfLoaded, blnIsTrain=False):
+    if (m_blnTrace2On):  print("INFO (claims.doFeatEng):  blnIsTrain, ", blnIsTrain)
+    
+    #--- remove cols
+    aryColsToDrop = ['BeneID', 'ClaimID', 'ClaimStartDt','ClaimEndDt','AttendingPhysician',
+                     'OperatingPhysician', 'OtherPhysician', 'ClmDiagnosisCode_1',
+                     'ClmDiagnosisCode_2', 'ClmDiagnosisCode_3', 'ClmDiagnosisCode_4',
+                     'ClmDiagnosisCode_5', 'ClmDiagnosisCode_6', 'ClmDiagnosisCode_7',
+                     'ClmDiagnosisCode_8', 'ClmDiagnosisCode_9', 'ClmDiagnosisCode_10',
+                     'ClmProcedureCode_1', 'ClmProcedureCode_2', 'ClmProcedureCode_3',
+                     'ClmProcedureCode_4', 'ClmProcedureCode_5', 'ClmProcedureCode_6',
+                     'ClmAdmitDiagnosisCode', 'AdmissionDt',
+                     'DischargeDt', 'DiagnosisGroupCode','DOB', 'DOD',
+                     'State', 'County']
+    pdfFeatEng = pdfLoaded.drop(columns=aryColsToDrop, axis=1)
+
+    #--- flag categorical cols
+    pdfFeatEng.Gender = pdfFeatEng.Gender.astype('category')
+    pdfFeatEng.Race = pdfFeatEng.Race.astype('category')
+
+    #--- one-hot-encoding
+    pdfFeatEng = pd.get_dummies(pdfFeatEng, columns=['Gender', 'Race'], drop_first=True)
+    if (blnIsTrain):
+        #--- one-hot encode the potential fraud column (for training data only)
+        try:
+            #print("INFO (claims.doFeatEng):  one-hot encoding potential fraud")
+            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'Yes', 'PotentialFraud'] = 1
+            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'No', 'PotentialFraud'] = 0
+        except KeyError:
+            #--- likely column not found; invalid fxn call
+            print("ERROR (claims.doFeatEng):  Potential Fraud col not found")
+
+    pdfFeatEng.loc[pdfFeatEng['RenalDiseaseIndicator'] == 'Y', 'RenalDiseaseIndicator'] = 1 
+    pdfFeatEng['DeductibleAmtPaid'].fillna(0, inplace=True)
+    pdfFeatEng['AdmittedDays'].fillna(0, inplace=True)
+
+    #--- check for correlated cols
+
+    #--- add new features to assist with predictions
+    pdfFeatEng['InscClaimReimbursement_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['InscClaimAmtReimbursed'].transform('mean') 
+    pdfFeatEng['DeductibleAmtPaid_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['DeductibleAmtPaid'].transform('mean')
+    
+    pdfFeatEng['IPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualReimbursementAmt'].transform('mean')
+    pdfFeatEng['IPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualDeductibleAmt'].transform('mean')
+
+    pdfFeatEng['OPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualReimbursementAmt'].transform('mean')
+    pdfFeatEng['OPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualDeductibleAmt'].transform('mean')
+    return pdfFeatEng
+
+
+
+#--- data eng on inpatient data
+def prep_inpatData(pdfInpat):
+    #--- calc admitted days
+    pdfInpat['AdmissionDt'] = pd.to_datetime(pdfInpat['AdmissionDt'], format='%Y-%m-%d')
+    pdfInpat['DischargeDt'] = pd.to_datetime(pdfInpat['DischargeDt'], format='%Y-%m-%d')
+    pdfInpat['AdmittedDays'] = round((pdfInpat['DischargeDt'] - pdfInpat['AdmissionDt']).dt.days + 1)
+    return pdfInpat
+
+
+
+#--- data eng on beneficiary data
+def prep_benefData(pdfBenef):
+    #--- chronic condition cols;  change any vals of 2 to 0 
+    aryCols = ['ChronicCond_Alzheimer', 'ChronicCond_Heartfailure', 
+                'ChronicCond_KidneyDisease', 'ChronicCond_Cancer',
+                'ChronicCond_ObstrPulmonary', 'ChronicCond_Depression',
+                'ChronicCond_Diabetes', 'ChronicCond_IschemicHeart',
+                'ChronicCond_Osteoporasis', 'ChronicCond_rheumatoidarthritis',
+                'ChronicCond_stroke'] 
+
+    for strVal in aryCols:
+        pdfBenef.replace({strVal: 2}, 0, inplace=True)
+
+    #--- fill missing data:  persons age
+    kstrDatetime = '2019-12-01'                            #--- the est datetime for the dataset
+    pdfBenef['DOB'] = pd.to_datetime(pdfBenef['DOB'], format = '%Y-%m-%d')
+    pdfBenef['DOD'] = pd.to_datetime(pdfBenef['DOD'], format = '%Y-%m-%d') 
+    pdfBenef['Age'] = round((pdfBenef['DOD'] - pdfBenef['DOB']).dt.days/365)
+    pdfBenef['Age'].fillna(round(((pd.to_datetime(kstrDatetime, format='%Y-%m-%d') - pdfBenef['DOB']).dt.days)/365), inplace=True)
+
+    #--- add an isDead flag column
+    pdfBenef.loc[pdfBenef['DOD'].isna(), 'DeadOrNot'] = 0 
+    pdfBenef.loc[pdfBenef['DOD'].notna(), 'DeadOrNot'] = 1 
+
+    return pdfBenef
+
+
+
+
+
+
+def get_kmeansPredict(pdfTestClaims):
+
+    #--- load test data
+    pdfClaims = pdfTestClaims
+    #print("INFO (claims.get_kmeansPredict)  pdfClaims.shape):  ", pdfClaims.shape)
+
+    #--- perform featEng, std scaling
+    print("TRACE: claims.kmeansPredict  perform featEng, stdScaling ...")
+    pdfFeatEng = mdl_kmeans.do_featEng(pdfClaims, False, False)
+    npaScaled = mdl_utils.doClaims_stdScaler(pdfFeatEng, False)
+    pdfScaled = mdl_utils.doClaims_stdScaler_toPdf(npaScaled)
+    #print("INFO (predict.npaScaled.shape):  ", npaScaled.shape)
+
+    #--- get the pre-fit kmeans clusters
+    #--- predict/label clusters against data points 
+    print("TRACE: claims.kmeansPredict  perform kmeans predict ...")
+    ndaPredict = mdl_kmeans.predict(pdfScaled)
+    #print("INFO (predict.npaPredict.shape):  ", ndaPredict.shape)
+
+    pdfPredict = pd.DataFrame(ndaPredict)
+    #print("INFO (predict.pdfPredict.shape):  ", pdfPredict.shape)
+
+    #--- stitch the data with the labels
+    print("TRACE: claims.kmeansPredict  stitch labels with results ...")
+    pdfResults = pdfTestClaims
+    #print("INFO (predict.pdfGrpFeatEng.shape):  ", pdfResults.shape)
+
+    pdfResults.insert(0, "cluster", pdfPredict[0])
+    return pdfResults
+
+
+
+def get_kmeansFit(pdfTestClaims):
+
+    pdfClaims = pdfTestClaims
+    pdfFeatEng = do_featEng(pdfClaims, False)                           #--- not grouped by provider
+    
+
+    #--- perform standard scaling; get fit then transform                      
+    npaScaled = mdl_utils.do_stdScaler(pdfFeatEng, False)               #--- grouped by provider
+    pdfScaled = mdl_utils.do_stdScaler_toPdf(npaScaled)
+    #print("INFO (predict.npaScaled.shape):  ", npaScaled.shape)
+
+    #--- SKIP:  perform PCA; then kmeans fit (this was done to determine the KMeans params)
+    #--- get Kmeans object, instantiated with trained args, and fit to test/prod scaled data
+    #--- OR ... assume that the kmeans is already fit, and we now want to predict which cluster each data point appears in
+    mdlKmeans = mdl_kmeans.fit(pdfScaled)
+    """ 
+
+        pdfPredict = pd.DataFrame(ndaPredict)
+        #print("INFO (predict.pdfPredict.shape):  ", pdfPredict.shape)
+
+        #--- stitch the grouped data with the labels
+        pdfResults = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+        #print("INFO (predict.pdfGrpFeatEng.shape):  ", pdfResults.shape)
+
+        pdfResults.insert(0, "hasAnom?", pdfPredict[0]) 
+
+        Notes:
+        - train_final = trainAllPatientDetailsProvider          #--- ungrouped data (558211, 27); has PotentialFraud cols
+        - train_final = pd.get_dummies                          #--- post one-hot encoding (558211, 25=27-2+4) ;  -Gender-Race + 4*(Gender+Race one-hot encoding) 
+        - y, X:  X.shape = (558211, 27);  y.shape=(558211,1)    #--- X popped PotentialFraud, and dropped Provider
+        - train_final[cluster_labels] = mdlKMeans.labels
+    """
+
+    return mdlKmeans    

lib/models/mdl_autoenc.py

@@ -0,0 +1,55 @@
+import pandas as pd
+import numpy as np
+from sklearn.decomposition import PCA 
+import lib.utils as libPaths
+import pickle
+
+
+m_kstrFile = __file__
+m_kstrDataPath = libPaths.pth_data
+m_kstrBinModelPath = libPaths.pth_binModels
+m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_unsuperv_colab.pkl'
+m_kstrEncModelPath = m_kstrBinModelPath + 'enc_keras_seq/'
+
+
+#--- Supervised:  autoencoder - Principal Component Analysis
+def load_encFromKeras():
+    from tensorflow import keras
+    mdlAnoms = keras.models.load_model(m_kstrEncModelPath)
+    return mdlAnoms
+
+
+def load_pcaFromPkl():
+    with open(m_kstrPcaModelPath, 'rb') as filPkl:
+        # load using pickle de-serializer
+        mdlAnoms = pickle.load(filPkl)
+    return mdlAnoms
+
+
+def save_encToKeras(mdlAnoms):
+    mdlAnoms.save(m_kstrEncModelPath)
+
+
+
+def predict(pdfScaled):
+    
+    #--- Pre:  Transforming train and test dataframes based on PCA
+    mdlPCA = load_pcaFromPkl()         #--- this is a pre-fit model based on training
+    npaPca = mdlPCA.transform(pdfScaled)
+    print("INFO (" + m_kstrFile + ".predict)  npaPca.shape:  ", npaPca.shape)
+
+
+    #--- predict on unseen data
+    mdlEnc = load_encFromKeras()
+    npaPredict = mdlEnc.predict(npaPca[:,:29])
+    print("INFO (" + m_kstrFile + ".predict)  npaPredict.shape:  ", npaPredict.shape)
+    #--- expected:  297, 29?
+    return npaPredict
+    
+
+""" 
+def train(pdfTrainData):
+    mdlAnoms = PCA()                        #---- TODO:  this is Keras Sequential
+    mdlAnoms.fit(pdfTrainData.values)
+    save_encToKeras(mdlAnoms)
+    return mdlAnoms """

lib/models/mdl_kmeans.py

@@ -0,0 +1,155 @@
+from sklearn.cluster import KMeans
+import lib.utils as libPaths
+import pickle
+import pandas as pd
+
+
+m_kstrFile = __file__
+m_kstrDataPath = libPaths.pth_data
+m_kstrBinModelPath = libPaths.pth_binModels
+
+#m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_kmeans_unsuperv_colab.pkl'
+#m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_kmeans_unsuperv_colab_v1.2.1.pkl'
+m_kstrPcaModelPath_111 = m_kstrBinModelPath + 'claims_pca_v1.1.1_27cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
+m_kstrPcaModelPath_121 = m_kstrBinModelPath + 'claims_pca_v1.2.1_27cols.pkl'
+m_kstrPcaModelPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_pca_py3816_sk111hp_27cols.pkl'
+m_kstrPcaModelPath = m_kstrPcaModelPath_claims_py3816_sk111hp
+
+#m_kstrKmeansModelPath = m_kstrBinModelPath + 'kmeans_unsuperv_colab.pkl'
+#m_kstrKmeansModelPath = m_kstrBinModelPath + 'kmn_unsuperv_colab_v1.2.1.pkl'
+m_kstrModelPath_111 = m_kstrBinModelPath + 'claims_kmn_v1.1.1_22cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
+m_kstrModelPath_121 = m_kstrBinModelPath + 'claims_kmn_v1.2.1_22cols.pkl'
+m_kstrModelPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_kmn_py3816_sk111hp_22cols.pkl'
+m_kstrKmeansModelPath = m_kstrModelPath_claims_py3816_sk111hp
+
+m_blnTraceOn = True
+
+
+#--- unsupervised:  Logistic Regession
+def load_pcaFromPkl():
+    with open(m_kstrPcaModelPath, 'rb') as filPkl:
+        mdlAnoms = pickle.load(filPkl)
+    return mdlAnoms
+
+
+#--- unsupervised:  KMeans
+def load_kmeansFromPkl():
+    with open(m_kstrKmeansModelPath, 'rb') as filPkl:
+        mdlAnoms = pickle.load(filPkl)
+    return mdlAnoms
+
+
+def save_pcaToPkl(mdlAnoms):
+    with open(m_kstrPcaModelPath, 'wb') as filPkl:
+        pickle.dump(mdlAnoms, filPkl)
+    return mdlAnoms
+
+
+def save_kmeansToPkl(mdlAnoms):
+    with open(m_kstrKmeansModelPath, 'wb') as filPkl:
+        pickle.dump(mdlAnoms, filPkl)
+    return mdlAnoms
+
+
+
+#--- determine which points can be labelled against which clusters
+def predict(pdfScaled):
+    #--- load a persisted fit kmeans model
+    #--- predict will assign labels onto a similarly scaled data frame 
+
+
+    #--- Note:  reverse chron through the code ...
+    #---        4. KMeans was fit on X-reduced (22 cols)
+    #---        3. X_reduced was a reduced column set of X-scaled (27 -> 22;  Dropped 5 cols:  DeadOrNot; and hotEncoded Gender and Race)
+    #---        2. x_scaled was transformed through stdScaler
+    #---        1. StdScaler was fit on X to produce X-scaled (X has 27 cols)
+    pdfReduced = pdfScaled[['InscClaimAmtReimbursed', 'DeductibleAmtPaid',
+        'AdmittedDays', 'RenalDiseaseIndicator', 'NoOfMonths_PartACov',
+        'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
+        'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
+        'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
+        'ChronicCond_Depression', 'ChronicCond_Diabetes',
+        'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
+        'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
+        'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
+        'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age']]
+
+    #--- prefit Kmeans clustering - was fit on trained pdfReduced
+    #--- Note:  if we want to understand how kmeans performs on test/prod data, we need to predict
+    mdlKMeans = load_kmeansFromPkl()
+    #ndaPredict = mdlKMeans.predict(pdfScaled)         #20230208:  ValueError: X has 27 features, but KMeans is expecting 22 features as input.
+    ndaPredict = mdlKMeans.predict(pdfReduced)       #ValueError: X has 22 features, but KMeans is expecting 27 features as input.
+    return ndaPredict
+
+
+#--- feat eng
+def do_featEng(pdfLoaded, blnIsTrain=False, hasGroupByProviderCols=True):
+    print("INFO (mdl_kmeans.doFeatEng):  blnIsTrain, ", blnIsTrain)
+
+    #--- columns_to_remove
+    aryColsToDrop = ['BeneID', 'ClaimID', 'ClaimStartDt','ClaimEndDt','AttendingPhysician',
+                     'OperatingPhysician', 'OtherPhysician', 'ClmDiagnosisCode_1',
+                     'ClmDiagnosisCode_2', 'ClmDiagnosisCode_3', 'ClmDiagnosisCode_4',
+                     'ClmDiagnosisCode_5', 'ClmDiagnosisCode_6', 'ClmDiagnosisCode_7',
+                     'ClmDiagnosisCode_8', 'ClmDiagnosisCode_9', 'ClmDiagnosisCode_10',
+                     'ClmProcedureCode_1', 'ClmProcedureCode_2', 'ClmProcedureCode_3',
+                     'ClmProcedureCode_4', 'ClmProcedureCode_5', 'ClmProcedureCode_6',
+                     'ClmAdmitDiagnosisCode', 'AdmissionDt',
+                     'DischargeDt', 'DiagnosisGroupCode','DOB', 'DOD',
+                     'State', 'County']
+    pdfFeatEng = pdfLoaded.drop(columns=aryColsToDrop, axis=1)
+
+    #--- flag categorical cols
+    pdfFeatEng.Gender = pdfFeatEng.Gender.astype('category')
+    pdfFeatEng.Race = pdfFeatEng.Race.astype('category')
+
+    #--- one-hot-encoding
+    pdfFeatEng = pd.get_dummies(pdfFeatEng, columns=['Gender', 'Race'], drop_first=True)
+    if (blnIsTrain):
+        #--- one-hot encode the potential fraud column (for training data only)
+        try:
+            #print("INFO (claims.doFeatEng):  one-hot encoding potential fraud")
+            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'Yes', 'PotentialFraud'] = 1
+            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'No', 'PotentialFraud'] = 0
+        except KeyError:
+            #--- likely column not found; invalid fxn call
+            print("ERROR (claims.doFeatEng):  Potential Fraud col not found")
+
+    pdfFeatEng.loc[pdfFeatEng['RenalDiseaseIndicator'] == 'Y', 'RenalDiseaseIndicator'] = 1 
+    pdfFeatEng['DeductibleAmtPaid'].fillna(0, inplace=True)
+    pdfFeatEng['AdmittedDays'].fillna(0, inplace=True)
+
+    #--- check for correlated cols
+
+    #--- add new features to assist with predictions
+    if (hasGroupByProviderCols):
+        pdfFeatEng['InscClaimReimbursement_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['InscClaimAmtReimbursed'].transform('mean') 
+        pdfFeatEng['DeductibleAmtPaid_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['DeductibleAmtPaid'].transform('mean')
+        
+        pdfFeatEng['IPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualReimbursementAmt'].transform('mean')
+        pdfFeatEng['IPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualDeductibleAmt'].transform('mean')
+
+        pdfFeatEng['OPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualReimbursementAmt'].transform('mean')
+        pdfFeatEng['OPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualDeductibleAmt'].transform('mean')
+    return pdfFeatEng
+
+
+def fit(pdfScaled):
+    #--- determine the centroids of the kmeans clusters
+    #--- refit kmeans clustering according to the pre-scaled data provided
+    #--- note:  this all assumes that the nature of the data and the number of clusters remain unchanged
+    m_klngNumClusters = 3
+    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit)  instantiate KMeans ...")
+    mdlKMeans = KMeans(n_clusters=m_klngNumClusters, max_iter=50, random_state=2022)            #--- #clusters was learned from training    
+    
+    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit)  fitting data (scaled) ...")
+    mdlKMeans.fit(pdfScaled)           #--- fit on test/prod data
+    
+    return mdlKMeans                    #--- this ibject will give us all results based on kmeans
+
+
+def train(pdfTrainData):
+    mdlAnoms = KMeans(n_clusters=3, max_iter=50, random_state=2022)
+    mdlAnoms.fit(pdfTrainData.values)
+    save_kmeansToPkl(mdlAnoms)
+    return mdlAnoms

lib/models/mdl_logR.py

@@ -0,0 +1,41 @@
+from sklearn.linear_model import LogisticRegressionCV
+import lib.utils as libPaths
+import pickle
+
+
+m_kstrFile = __file__
+m_kstrDataPath = libPaths.pth_data
+m_kstrBinModelPath = libPaths.pth_binModels
+m_kstrModelPath = m_kstrBinModelPath + 'lgr_model_colab.pkl'
+
+
+#--- Supervised:  Logistic Regession
+def load_fromPkl():
+    with open(m_kstrModelPath, 'rb') as filPkl:
+        mdlAnoms = pickle.load(filPkl)
+    return mdlAnoms
+
+
+
+def save_toPkl(mdlAnoms):
+    with open(m_kstrModelPath, 'wb') as filPkl:
+        pickle.dump(mdlAnoms, filPkl)
+    return mdlAnoms
+
+
+
+def predict(npaData):
+    #--- input:  numpy.ndarray of feature eng, and scaled data 
+    mdlAnoms = load_fromPkl()
+    npaPredict = mdlAnoms.predict(npaData)
+
+    print("INFO (npaPredict.shape):  ", npaPredict.shape)
+    return npaPredict
+
+
+
+def train(pdfTrainData):
+    mdlAnoms = LogisticRegressionCV()
+    mdlAnoms.fit(pdfTrainData.values)
+    save_toPkl(mdlAnoms)
+    return mdlAnoms

lib/models/mdl_svm.py

@@ -0,0 +1,40 @@
+from sklearn.svm import LinearSVC
+import lib.utils as libPaths
+import pickle
+
+
+m_kstrFile = __file__
+m_kstrDataPath = libPaths.pth_data
+m_kstrBinModelPath = libPaths.pth_binModels
+m_kstrModelPath = m_kstrBinModelPath + 'svm_model_colab.pkl'
+
+
+#--- Supervised:  Support Vector Machines
+def load_fromPkl():
+    with open(m_kstrModelPath, 'rb') as filPkl:
+        mdlAnoms = pickle.load(filPkl)
+    return mdlAnoms
+
+
+
+def save_toPkl(mdlAnoms):
+    with open(m_kstrModelPath, 'wb') as filPkl:
+        pickle.dump(mdlAnoms, filPkl)
+    return mdlAnoms
+
+    
+
+def predict(npaData):
+    #--- input:  numpy.ndarray of feature eng, and scaled data 
+    mdlAnoms = load_fromPkl()
+    npaPredict = mdlAnoms.predict(npaData)
+    print("INFO (" + m_kstrFile + ".predict)  npaPredict.shape:  ", npaPredict.shape)
+    return npaPredict
+    
+
+
+def train(pdfTrainData):
+    mdlAnoms = LinearSVC()
+    mdlAnoms.fit(pdfTrainData.values)
+    save_toPkl(mdlAnoms)
+    return mdlAnoms

lib/models/mdl_utils.py

@@ -0,0 +1,256 @@
+import pandas as pd
+import pickle
+import lib.utils as libPaths
+
+m_blnTraceOn = False
+
+#--- load, merge data from file
+m_kstrDataPath = libPaths.pth_data
+m_kstrModelPath = libPaths.pth_model
+m_kstrBinModelPath = libPaths.pth_binModels
+
+#m_kstrScalerPath_claims = m_kstrBinModelPath + 'stdClaims_scaler_colab.pkl'         #--- does not work for scaling claims data;  from v1.0.2; using 1.1.1
+#m_kstrScalerPath_claims2 = m_kstrBinModelPath + 'std_scaler_unsuperv_colab.pkl'     #--- does not work; expects 32 features 
+#m_kstrScalerPath_claims = m_kstrBinModelPath + 'stdClaims_scaler_colab_v1.2.1.pkl'   
+m_kstrScalerPath_claims111 = m_kstrBinModelPath + 'claims_stdScaler_v1.1.1_27cols.pkl'
+m_kstrScalerPath_claims121 = m_kstrBinModelPath + 'claims_stdScaler_v1.2.1_27cols.pkl'
+m_kstrScalerPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_stdScl_py3816_sk111hp_27cols.pkl'
+m_kstrScalerPath_claims = m_kstrScalerPath_claims_py3816_sk111hp
+
+m_kstrScalerPath_providers111 = m_kstrBinModelPath + 'prov_stdScaler_v1.1.1_32cols.pkl'
+m_kstrScalerPath_providers121 = m_kstrBinModelPath + 'prov_stdScaler_v1.2.1_32cols.pkl'
+m_kstrScalerPath_prov_py3816_sk111 = m_kstrBinModelPath + 'prov_stdScl_py3816_sk111_32cols.pkl'
+m_kstrScalerPath_prov_py3816_sk111hp = m_kstrBinModelPath + 'prov_stdScl_py3816_sk111hp_32cols.pkl'
+m_kstrScalerPath_prov = m_kstrScalerPath_prov_py3816_sk111hp
+
+m_kstrScalerPath_providers_superv = m_kstrBinModelPath + 'gbc_scaler.pkl'
+m_kstrScalerPath_providers_train = m_kstrBinModelPath + "stdProvider_scaler.pkl" 
+   
+
+
+def doProviders_stdScaler(pdfFeatEng, blnIsTrain=False, hasGroupByProviderCols=True):
+    print("INFO (claims.do_stdScaler):  blnIsTrain, ", blnIsTrain)
+
+    #--- Note:  prediction runs on X_val
+    '''
+    #--- WARN:  The default value of numeric_only in DataFrameGroupBy.sum is deprecated. 
+    #           In a future version, numeric_only will default to False. Either specify 
+    #           numeric_only or select only columns which should be valid for the function.
+    '''
+
+    #--- WARN:  this code groups all data by provider;  any predictions will also be by provider
+    pdfGroupBy = pdfFeatEng
+    if (hasGroupByProviderCols):
+        pdfGroupBy = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+    
+    X = pdfGroupBy
+
+    try:
+        X = X.drop(columns=['Provider'], axis=1)        #--- cannot scale;  text
+    except KeyError:
+        #--- likely column not found; invalid fxn call
+        print("ERROR (mdlUtils.doProviders_stdScaler):  Provider col not found")
+
+    try:
+        X = X.drop(columns=['PotentialFraud'], axis=1)  
+    except KeyError:
+        #--- likely column not found; invalid fxn call
+        if (blnIsTrain):  print("ERROR (mdlUtils.doProviders_stdScaler):  Potential Fraud col not found")
+
+
+    #--- apply std scaler
+    #--- WARN:  scaling is also grouped by provider
+    if (m_blnTraceOn):  print("INFO (mdlUtils.doProviders_stdScaler)  cols: ", X.columns)           #--- 32cols
+    X_std = fitProviders_txfStdScaler(X, blnIsTrain)
+    return X_std
+
+
+
+def doClaims_stdScaler(pdfFeatEng, blnIsTrain=False):
+    print("INFO (mdlUtils.doClaims_stdScaler):  blnIsTrain, ", blnIsTrain)
+
+    #--- Note:  prediction runs on X_val
+    '''
+    #--- WARN:  The default value of numeric_only in DataFrameGroupBy.sum is deprecated. 
+    #           In a future version, numeric_only will default to False. Either specify 
+    #           numeric_only or select only columns which should be valid for the function.
+    '''
+
+    #--- WARN:  this code groups all data by provider;  any predictions will also be by provider
+    X = pdfFeatEng
+
+    try:
+        X = X.drop(columns=['Provider'], axis=1)        #--- cannot scale;  text
+    except KeyError:
+        #--- likely column not found; invalid fxn call
+        print("ERROR (mdlUtils.do_stdScaler):  Provider col not found")
+
+    try:
+        X = X.drop(columns=['PotentialFraud'], axis=1)  
+    except KeyError:
+        #--- likely column not found; invalid fxn call
+        if (blnIsTrain):  print("ERROR (mdlUtils.do_stdScaler):  Potential Fraud col not found")
+
+
+    #--- apply std scaler
+    #--- WARN:  scaling is also grouped by provider
+    #print("INFO (mdlUtils.doClaims_stdScaler)  cols: ", X.columns)
+    X_std = fitClaims_txfStdScaler(X, blnIsTrain)
+    return X_std
+
+
+
+def doProviders_stdScaler_toPdf(npaScaled):
+    #--- NOTE:  the list of cols came from doProvider_stdScaler; print(X.columns)
+    aryCols = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'AdmittedDays',
+       'NoOfMonths_PartACov', 'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
+       'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
+       'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
+       'ChronicCond_Depression', 'ChronicCond_Diabetes',
+       'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
+       'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
+       'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
+       'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age', 'DeadOrNot',
+       'Gender_2', 'Race_2', 'Race_3', 'Race_5',
+       'ClaimReimbursement_ProviderAvg',
+       'ClaimReimbursement_AttendingPhysician',
+       'ClaimReimbursement_OperatingPhysician',
+       'DeductibleAmtPaid_ProviderAvg', 'DeductibleAmtPaid_AttendingPhysician',
+       'DeductibleAmtPaid_OperatingPhysician']
+
+    #npaScaled = do_stdScaler(pdfFeatEng)
+    pdfScaled = pd.DataFrame(npaScaled, columns=aryCols)
+    return pdfScaled
+
+
+
+def doClaims_stdScaler_toPdf(npaScaled):
+    #--- NOTE:  the list of cols came from doClaims_stdScaler; print(X.columns)
+    aryCols = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'AdmittedDays',
+       'RenalDiseaseIndicator', 'NoOfMonths_PartACov', 'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
+       'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
+       'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
+       'ChronicCond_Depression', 'ChronicCond_Diabetes',
+       'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
+       'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
+       'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
+       'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age', 'DeadOrNot',
+       'Gender_2', 'Race_2', 'Race_3', 'Race_5']
+
+    #npaScaled = do_stdScaler(pdfFeatEng)
+    pdfScaled = pd.DataFrame(npaScaled, columns=aryCols)
+    return pdfScaled
+
+
+
+
+def fitClaims_stdScaler(pdfData, blnIsTrain=False):
+    #--- apply scaler
+    #--- WARN:  scaling is not grouped by provider
+    from sklearn.preprocessing import StandardScaler
+
+    #--- note:  this is a numpy.ndarray
+    #--- we need to fit the scaler, and then save as a pkl file
+    #strScalerPath = m_kstrScalerPath_claims
+    strScalerPath = m_kstrScalerPath_claims
+#    strScalerPath = m_kstrBinModelPath + "stdClaims_scaler_colab.pkl"
+    if (m_blnTraceOn):  print("INFO (lib.model.fitClaims_stdScalar):  ", strScalerPath)
+    if (blnIsTrain):
+        scaler = StandardScaler()
+        sclFit = scaler.fit(pdfData)
+        #--- if we train locally;  write out to gbc_scalar.pkl
+        #--- we do not want to overwrite the colab version used for test
+        strScalerPath = m_kstrBinModelPath + "stdClaims_scaler.pkl"
+        if (m_blnTraceOn):  print("INFO (lib.model.fit_stdScalar)  Using local pkl for Train: ", strScalerPath)
+        with open(strScalerPath, 'wb') as filPkl:
+            pickle.dump(sclFit, filPkl)
+    else:
+        #--- we need to load the pkl file
+        import sklearn
+        if (m_blnTraceOn):  print("INFO (lib.model.fit_stdScalar)  Using colab pkl for Test: ", strScalerPath)
+        with open(strScalerPath, 'rb') as filPkl:
+            sclFit = pickle.load(filPkl)
+        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sclFit.type: ", type(sclFit))
+
+        #--- testing
+        scaler = StandardScaler()
+        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  StdScaler.version: ", scaler.__getstate__()['_sklearn_version'])
+        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sclFit.version: " , sclFit.__getstate__()['_sklearn_version'])
+        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sklearn.version: " , sklearn.__version__)
+    return sclFit
+
+
+
+def fitProviders_stdScaler(pdfData, blnIsTrain=False):
+    #--- apply scaler
+    #--- WARN:  scaling is also grouped by provider
+    from sklearn.preprocessing import StandardScaler
+
+    #--- note:  this is a numpy.ndarray
+    #--- we need to fit the scaler, and then save as a pkl file
+    #strScalerPath = m_kstrScalerPath_providers
+    #strScalerPath = m_kstrScalerPath_providers_train            
+    strScalerPath = m_kstrScalerPath_prov         
+    print("INFO (libModel.fitProviders_stdScalar):  ", strScalerPath)
+    if (blnIsTrain):
+        scaler = StandardScaler()
+        sclFit = scaler.fit(pdfData)
+        #--- if we train locally;  write out to gbc_scalar.pkl
+        #--- we do not want to overwrite the colab version used for test
+        strScalerPath = m_kstrScalerPath_providers_train       #--- works for provider training
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using local pkl for Train: ", strScalerPath)
+        with open(strScalerPath, 'wb') as filPkl:
+            pickle.dump(sclFit, filPkl)
+    else:
+        #--- we need to load the pkl file
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using colab pkl for Test: ", strScalerPath)
+        with open(strScalerPath, 'rb') as filPkl:
+            sclFit = pickle.load(filPkl)
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  sclFit.type: ", type(sclFit))
+    return sclFit
+
+
+
+def fitProviders_stdScalerSuperv(pdfData, blnIsTrain=False):
+    #--- apply scaler
+    #--- WARN:  scaling is also grouped by provider
+    from sklearn.preprocessing import StandardScaler
+
+    #--- note:  this is a numpy.ndarray
+    #--- we need to fit the scaler, and then save as a pkl file
+    strScalerPath = m_kstrScalerPath_prov
+    if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar):  ", strScalerPath)
+    if (blnIsTrain):
+        scaler = StandardScaler()
+        sclFit = scaler.fit(pdfData)
+        #--- if we train locally;  write out to gbc_scalar.pkl
+        #--- we do not want to overwrite the colab version used for test
+        strScalerPath = m_kstrBinModelPath + "stdProvider_scaler.pkl"
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using local pkl for Train: ", strScalerPath)
+        with open(strScalerPath, 'wb') as filPkl:
+            pickle.dump(sclFit, filPkl)
+    else:
+        #--- we need to load the pkl file
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using colab pkl for Test: ", strScalerPath)
+        with open(strScalerPath, 'rb') as filPkl:
+            sclFit = pickle.load(filPkl)
+        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  sclFit.type: ", type(sclFit))
+    return sclFit
+
+
+
+def fitProviders_txfStdScaler(pdfData, blnIsTrain=False):
+    from sklearn.preprocessing import StandardScaler
+    sclFit = fitProviders_stdScaler(pdfData, blnIsTrain)
+    X_std = sclFit.transform(pdfData)
+    return X_std
+
+
+
+def fitClaims_txfStdScaler(pdfData, blnIsTrain=False):
+    from sklearn.preprocessing import StandardScaler
+    sclFit = fitClaims_stdScaler(pdfData, blnIsTrain)
+
+
+    X_std = sclFit.transform(pdfData)
+    return X_std

lib/models/mdl_xgb.py

@@ -0,0 +1,66 @@
+import pandas as pd
+from sklearn.ensemble import GradientBoostingClassifier
+import lib.utils as libPaths
+import pickle
+import sys
+
+
+m_kstrFile = __file__
+m_kstrDataPath = libPaths.pth_data
+m_kstrBinModelPath = libPaths.pth_binModels
+m_kstrModelPath_gbc = m_kstrBinModelPath + 'gbc_model_colab.pkl'
+m_kstrModelPath_prov111 = m_kstrBinModelPath + 'prov_gbc_v1.1.1_32cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
+m_kstrModelPath_prov121 = m_kstrBinModelPath + 'prov_gbc_v1.2.1_32cols.pkl'
+m_kstrModelPath_prov_py3816_sk111hp = m_kstrBinModelPath + 'prov_gbc_py3816_sk111hp_32cols.pkl'
+m_kstrModelPath = m_kstrModelPath_prov_py3816_sk111hp
+
+m_blnTraceOn = True
+
+
+
+#--- Supervised:  xg boost;  gradient boosting classifier
+def load_fromPkl():
+    try:
+        with open(m_kstrModelPath, 'rb') as filPkl:
+            mdlAnoms = pickle.load(filPkl)
+        return mdlAnoms
+
+    except:
+        e = sys.exc_info()
+        print("ERROR (mdl_xgb.load_fromPkl_genError):  ", e)    
+
+
+
+def save_toPkl(mdlAnoms):
+    with open(m_kstrModelPath, 'wb') as filPkl:
+        pickle.dump(mdlAnoms, filPkl)
+    return mdlAnoms
+
+
+
+def predict(npaData):
+
+    try:
+        #--- input:  numpy.ndarray of feature eng, and scaled data 
+        mdlAnoms = load_fromPkl()
+        if (m_blnTraceOn):  print("TRACE (mdl_xgb.predict):  data loaded ... ")
+        npaPredict = mdlAnoms.predict(npaData)
+
+    except:
+        e = sys.exc_info()
+        print("ERROR (mdl_xgb.predict_genError1):  ", e)  
+
+
+    #--- AttributeError: 'GradientBoostingClassifier' object has no attribute '_loss'
+    #--- version of scikit-learn?  Monika: ?.?.? ; Iain: 1.2.0
+
+    #print("INFO (type.npaPredict):  ", type(npaPredict))
+    #if (m_blnTraceOn):  print("TRACE (mdl_xgb.predict) npaPredict.shape:  ", npaPredict.shape)
+    return npaPredict
+
+
+def train(pdfTrainData):
+    mdlAnoms = GradientBoostingClassifier()
+    mdlAnoms.fit(pdfTrainData.values)
+    save_toPkl(mdlAnoms)
+    return mdlAnoms

lib/providers.py

@@ -0,0 +1,170 @@
+import pandas as pd
+import lib.utils as libPaths
+import lib.claims as libClaims
+
+from lib.models import mdl_utils, mdl_xgb, mdl_logR, mdl_svm
+from lib.models import mdl_autoenc, mdl_kmeans
+import sys
+
+m_blnTraceOn = True
+m_blnTrace2On = False
+
+#--- load, merge data from file
+m_kstrDataPath = libPaths.pth_data
+m_kstrModelPath = libPaths.pth_model
+m_kstrBinModelPath = libPaths.pth_binModels
+
+
+
+def load_providers(blnIsTrain=False):
+
+    pdfClaims = libClaims.loadPkl_claims(blnIsTrain)
+    pdfClaims = pdfClaims.drop(['ClmProcedureCode_1', 'ClmProcedureCode_2', 'ClmProcedureCode_3', 
+                            'ClmProcedureCode_4', 'ClmProcedureCode_5', 'ClmProcedureCode_6',
+                            'Gender', 'Race', 'County'], axis=1)
+    pdfProviders = pdfClaims.groupby(['Provider'], as_index=False).agg('sum')
+    return pdfProviders
+
+
+#--- feat eng
+def do_featEng(pdfClaimsFeatEng, blnIsTrain=False):
+    if (m_blnTraceOn): print("TRACE (providers.doFeatEng):  blnIsTrain, ", blnIsTrain)
+    pdfFeatEng = pdfClaimsFeatEng
+
+    #--- add new features to assist with predictions
+    pdfFeatEng['InscClaimReimbursement_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['InscClaimAmtReimbursed'].transform('mean') 
+    pdfFeatEng['DeductibleAmtPaid_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['DeductibleAmtPaid'].transform('mean')
+    
+    pdfFeatEng['IPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualReimbursementAmt'].transform('mean')
+    pdfFeatEng['IPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualDeductibleAmt'].transform('mean')
+
+    pdfFeatEng['OPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualReimbursementAmt'].transform('mean')
+    pdfFeatEng['OPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualDeductibleAmt'].transform('mean')
+    return pdfFeatEng
+
+
+
+def get_logrPredict(pdfTestClaims):
+
+    #--- logistic regression predictions;  load test data
+    pdfClaims = pdfTestClaims
+    #print("INFO (providers.get_logrPredict)  pdfClaims.shape):  ", pdfClaims.shape)
+
+    pdfFeatEng = do_featEng(pdfClaims, False)
+    npaScaled = mdl_utils.doProviders_stdScaler(pdfFeatEng, False)
+    pdfScaled = mdl_utils.doProviders_stdScaler_toPdf(npaScaled)
+    #print("INFO (predict.npaScaled.shape):  ", npaScaled.shape)
+
+    ndaPredict = mdl_logR.predict(npaScaled)
+    #print("INFO (predict.npaPredict.shape):  ", ndaPredict.shape)
+
+    pdfPredict = pd.DataFrame(ndaPredict)
+    #print("INFO (predict.pdfPredict.shape):  ", pdfPredict.shape)
+
+    #--- stitch the grouped data with the labels
+    pdfResults = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+    #print("INFO (predict.pdfGrpFeatEng.shape):  ", pdfResults.shape)
+
+    pdfResults.insert(0, "hasAnom?", pdfPredict[0])
+    return pdfResults    
+
+
+def get_svmPredict(pdfTestClaims):
+
+    #--- support vector machine predictions;  load test data
+    pdfClaims = pdfTestClaims
+    if (m_blnTraceOn):  print("TRACE (providers.get_svmPredict) pdfClaims.shape:  ", pdfClaims.shape)
+
+    pdfFeatEng = do_featEng(pdfClaims, False)
+    npaScaled = mdl_utils.doProviders_stdScaler(pdfFeatEng, False)
+    pdfScaled = mdl_utils.doProviders_stdScaler_toPdf(npaScaled)
+    if (m_blnTraceOn):  print("TRACE (providers.get_svmPredict) npaScaled.shape:  ", npaScaled.shape)
+
+    ndaPredict = mdl_svm.predict(npaScaled)
+    if (m_blnTraceOn):  print("TRACE (providers.get_svmPredict) npaPredict.shape:  ", ndaPredict.shape)
+
+    pdfPredict = pd.DataFrame(ndaPredict)
+    if (m_blnTraceOn):  print("TRACE (providers.get_svmPredict) pdfPredict.shape:  ", pdfPredict.shape)
+
+    #--- stitch the grouped data with the labels
+    pdfResults = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+    if (m_blnTraceOn):  print("TRACE (providers.get_svmPredict) pdfResults.shape:  ", pdfResults.shape)
+
+    pdfResults.insert(0, "hasAnom?", pdfPredict[0])
+    return pdfResults    
+
+
+
+def get_xgbPredict(pdfTestClaims):
+
+    try:
+        #--- load test data
+        pdfClaims = pdfTestClaims
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict)  pdfClaims.shape):  ", pdfClaims.shape)
+
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) doFeatEng (provider) ... ")
+        pdfFeatEng = do_featEng(pdfClaims, False)
+
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) doStdScaler ... ")
+        npaScaled = mdl_utils.doProviders_stdScaler(pdfFeatEng, False)
+
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) doStdScaler_toPdf ... ")
+        pdfScaled = mdl_utils.doProviders_stdScaler_toPdf(npaScaled)
+        #if (m_blnTraceOn):  print("TRACE (predict.npaScaled.shape1):  ", npaScaled.shape)
+
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) run prediction ... ")
+        ndaPredict = mdl_xgb.predict(npaScaled)
+        #if (m_blnTraceOn):  print("TRACE (predict.npaPredict.shape2):  ", ndaPredict.shape)
+
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) convert to dataframe ... ")
+        pdfPredict = pd.DataFrame(ndaPredict)
+        pdfAnoms = pdfPredict[pdfPredict[0] > 0]
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) pdfPredict.shape:  ", pdfPredict.shape)
+        if (m_blnTraceOn):  print("TRACE (providers.get_xgbPredict) #anoms:  ", len(pdfAnoms.index))
+
+        #--- group data by provider
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) group claims by provider ... ")
+        pdfResults = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) pdfResults.shape:  ", pdfResults.shape)
+
+        #--- stitch the grouped data with the labels
+        if (m_blnTrace2On):  print("TRACE (providers.get_xgbPredict) merge labels into dataset ... ")
+        pdfResults.insert(0, "hasAnom?", pdfPredict[0])
+
+    except:
+        e = sys.exc_info()
+        print("ERROR (providers.get_xgbPredict_genError):  ", e)  
+    
+    
+    if (m_blnTraceOn):  print("TRACE (providers.get_xgbPredict) proc complete; return ... ")
+    return pdfResults
+
+
+
+def get_encPredict(pdfTestClaims):
+
+    #--- principal component analysis predictions;  load test data
+    pdfClaims = pdfTestClaims
+    if (m_blnTraceOn):  print("TRACE (providers.get_encPredict) ppdfClaims.shape:  ", pdfClaims.shape)
+
+    pdfFeatEng = do_featEng(pdfClaims, False)                           #--- not grouped by provider
+    
+
+    #--- perform standard scaling; get fit then transform                      
+    npaScaled = mdl_utils.doProviders_stdScaler(pdfFeatEng, False)               #--- grouped by provider
+    pdfScaled = mdl_utils.doProviders_stdScaler_toPdf(npaScaled)
+    #print("INFO (predict.npaScaled.shape):  ", npaScaled.shape)
+
+    #--- perform PCA; then autoencode predict
+    ndaPredict = mdl_autoenc.predict(pdfScaled)
+    #print("INFO (predict.npaPredict.shape):  ", ndaPredict.shape)
+
+    pdfPredict = pd.DataFrame(ndaPredict)
+    #print("INFO (predict.pdfPredict.shape):  ", pdfPredict.shape)
+
+    #--- stitch the grouped data with the labels
+    pdfResults = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
+    #print("INFO (predict.pdfGrpFeatEng.shape):  ", pdfResults.shape)
+
+    pdfResults.insert(0, "hasAnom?", pdfPredict[0])
+    return pdfResults    

lib/utils.py

@@ -0,0 +1,23 @@
+#--- note:  this file is loaded by fastapi and streamlit, 
+# #         so keep it independant of those libs
+
+from pathlib import Path
+
+pth_pwd = Path(__file__).resolve().parent
+pth_appRoot = pth_pwd.parent
+
+pth_root = str(pth_appRoot) + "/"
+pth_api = pth_root + "api/"
+pth_bin = pth_root + "bin/"
+pth_binModels = pth_root + "bin/models/"
+pth_data = pth_root + "data/"
+pth_lib = pth_root + "lib/"
+pth_libModels = pth_root + "models/"
+pth_model = pth_root + "model/"
+pth_qa = pth_root + "qa/"
+pth_routes = pth_root + "routes/"
+pth_templ = pth_root + "templ/"
+pth_uix = pth_root + "uix/"
+
+m_klngMaxRecords = 100
+m_klngSampleSize = 25

lit_index.py

@@ -0,0 +1,25 @@
+'''
+    toExecute:  (from root app folder) ... streamlit run lit_index.py
+'''
+import streamlit as st
+#from uix import lit_sidebar as lit_sideBar
+import uix.lit_sidebar as litSideBar
+
+
+#--- streamlit:  specify title and logo
+st.set_page_config(
+            page_title='Healthcare Claims - ML Anomaly Detection', 
+            #page_icon='https://cdn.freebiesupply.com/logos/thumbs/1x/nvidia-logo.png', 
+            layout="wide")
+st.header("Healthcare ML Claims Anomaly Detection")
+st.markdown('---')
+
+
+#--- streamlit:  add a sidebar 
+litSideBar.init()
+
+
+#if __name__ == '__main__':
+#    st.run("main:app", host="0.0.0.0", port=48300, reload=True)
+
+#aryPkg[moduleNames.index(page)].run()

main.py

@@ -0,0 +1,97 @@
+'''
+    purpose:      
+'''
+
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from fastapi import APIRouter, Request, Response
+from fastapi.templating import Jinja2Templates
+import uvicorn
+
+
+from lib import claims as libClaims, providers as libProviders
+import lib.utils as libUtils
+from lib.models import mdl_utils as libMdlUtils
+
+
+#--- imported route handlers
+from routes.api.rte_api import rteApi
+from routes.qa.rte_qa import rteQa
+from routes.qa.rte_claims import rteClaims
+from routes.qa.rte_providers import rteProv
+
+
+#--- fastAPI self doc descriptors
+description = """
+    Fourthbrain Capstone:  MLE10 Cohort
+    
+    The Healthcare Claims Anomaly API is provided to assist with
+
+    ## Claims Analysis
+    ## Supervised Provider Predictions - Anomaly Detection (XGBoost)
+    ## Unsupervised Claim Predictions - Anomaly Detection (KMeans Cluster)
+
+    You will be able to:
+    * Analyze Claims data
+    * Identify potential Provider Anomalies
+    * Idenitfy potential Claim Anomalies
+"""
+
+app = FastAPI(
+    title="App:  Healthcare Claims - Anomaly Detection",
+    description=description,
+    version="0.0.1",
+    terms_of_service="http://example.com/terms/",
+    contact={
+        "name": "Iain McKone",
+        "email": "iain.mckone@gmail.com",
+    },
+    license_info={
+        "name": "Apache 2.0",
+        "url": "https://www.apache.org/licenses/LICENSE-2.0.html",
+    },
+)
+
+
+#--- configure route handlers
+app.include_router(rteApi, prefix="/api")
+app.include_router(rteQa, prefix="/qa")
+app.include_router(rteClaims, prefix="/claims")
+app.include_router(rteProv, prefix="/providers")
+
+
+
+m_kstrPath_templ = libUtils.pth_templ
+m_templRef = Jinja2Templates(directory=str(m_kstrPath_templ))
+
+
+def get_jinja2Templ(request: Request, pdfResults, strParamTitle, lngNumRecords, blnIsTrain=False, blnIsSample=False):
+    lngNumRecords = min(lngNumRecords, libUtils.m_klngMaxRecords)
+    if (blnIsTrain):  strParamTitle = strParamTitle + " - Training Data"
+    if (not blnIsTrain):  strParamTitle = strParamTitle + " - Test Data"
+    if (blnIsSample):  lngNumRecords = libUtils.m_klngSampleSize
+    strParamTitle = strParamTitle + " - max " + str(lngNumRecords) + " rows"
+
+    pdfClaims = pdfResults.sample(lngNumRecords)
+    htmlClaims = pdfClaims.to_html(classes='table table-striped')
+    kstrTempl = 'templ_showDataframe.html'
+    jsonContext = {'request': request, 
+                'paramTitle': strParamTitle,
+                'paramDataframe': htmlClaims
+            }
+    result = m_templRef.TemplateResponse(kstrTempl, jsonContext)
+    return result
+
+
+#--- get main ui/ux entry point
+@app.get('/')
+def index():
+    return {
+        "message": "Landing page:  Capstone Healthcare Anomaly Detection"
+    }
+
+
+
+if __name__ == '__main__':
+    uvicorn.run("main:app", host="0.0.0.0", port=48300, reload=True)
+#CMD ["uvicorn", "main:app", "--host=0.0.0.0", "--reload"]

routes/api/rte_api.py

@@ -0,0 +1,67 @@
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import JSONResponse
+
+import pandas as pd
+import json
+
+import lib.claims as libClaims
+from lib.models import mdl_utils, mdl_xgb
+
+
+rteApi = APIRouter()
+
+
+#--- return json for claims data (merged)
+#--- note:  current is kaggle, but future could include from yyyymm filter
+@rteApi.get('/claims', response_class = JSONResponse)
+def api_getClaims(request: Request, response: Response):
+    pdfClaims = libClaims.load_claims()
+    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
+    result = json.loads(jsonSample)
+    return result
+
+
+#--- return json for featEng
+@rteApi.get('/claims/doFeatEng/', response_class = JSONResponse)
+def tst_claims_featEng():
+    pdfClaims = libClaims.load_claims()
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
+    result = json.loads(jsonSample)
+    return result
+
+
+@rteApi.get('/claims/doStdScaling/', response_class = JSONResponse)
+def tst_claims_stdScaling():
+    pdfClaims = libClaims.load_claims()
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    pdfScaled = mdl_utils.doClaims_stdScaler_toPdf(pdfFeatEng)
+
+    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
+    result = json.loads(jsonSample)
+    return result
+
+
+@rteApi.get('/claims/predict/superv', response_class = JSONResponse)
+@rteApi.get('/claims/predict/xgb', response_class = JSONResponse)
+def predict_xgb():
+    #--- load test data
+    pdfClaims = libClaims.load_claims()
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+
+    npaScaled = mdl_utils.do_stdScaler(pdfFeatEng)
+    pdfScaled = mdl_utils.do_stdScaler_toPdf(npaScaled)
+  
+    ndaPredict = mdl_xgb.predict(npaScaled)
+    pdfPredict = pd.DataFrame(ndaPredict)
+
+    #--- stitch the grouped data with the labels
+    pdfResults = pdfScaled.copy()
+    pdfResults.insert(0, "hasAnom?", pdfPredict[0])
+
+    #--- filter to only those rows that are flagged with an anomaly
+    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
+
+    jsonSample = pdfResults.head(50).to_json(orient="records", indent=4)
+    result = json.loads(jsonSample)
+    return result

routes/qa/rte_claims.py

@@ -0,0 +1,139 @@
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import HTMLResponse
+
+
+import main as libMain
+from lib import utils as libUtils, claims as libClaims
+from lib.models import mdl_utils as libMdlUtils
+
+
+m_kstrFile = __file__
+m_blnTraceOn = True
+
+m_kstrPath_templ = libUtils.pth_templ
+
+
+rteClaims = APIRouter()
+
+
+
+#--- get claims data
+def claims_loadData(request: Request, response: Response, blnIsTrain=False, blnIsSample=False, blnForceCsv=False):
+ 
+    pdfClaims = libClaims.load_claims(blnIsTrain)
+    lngNumRecords = libUtils.m_klngMaxRecords
+    strParamTitle = "Claims"
+
+    return libMain.get_jinja2Templ(request, pdfClaims, strParamTitle, lngNumRecords, blnIsTrain, blnIsSample)
+
+
+
+@rteClaims.get('/data/loadCsv/', response_class = HTMLResponse)
+def claims_loadCsv(request: Request, response: Response):
+    #--- forces a reload of csv's in case a refresh is required
+    pdfClaims = libClaims.load_claims(False, True)
+    pdfClaims = libClaims.load_claims(True, True)
+    return claims_loadData(request, response, True,False)
+
+
+
+@rteClaims.get('/data/train/', response_class = HTMLResponse)
+def claims_loadTrainData(request: Request, response: Response, blnIsSample=False):
+    return claims_loadData(request, response, True, blnIsSample)
+
+
+
+@rteClaims.get('/data/train/sample', response_class = HTMLResponse)
+def claims_loadTrainSample(request: Request, response: Response):
+    return claims_loadTrainData(request, response, True)
+
+
+
+@rteClaims.get('/data/test/', response_class = HTMLResponse)
+def claims_loadTestData(request: Request, response: Response, blnIsSample=False):
+    return claims_loadData(request, response, False, blnIsSample)
+
+
+
+@rteClaims.get('/data/test/sample', response_class = HTMLResponse)
+def claims_loadTestSample(request: Request, response: Response):
+    return claims_loadTestData(request, response, True)
+
+
+
+@rteClaims.get('/doStdScaling/', response_class = HTMLResponse)
+def claims_stdScaling(request: Request, response: Response, blnIsTrain=False):
+    pdfClaims = libClaims.load_claims(blnIsTrain)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims, blnIsTrain, False)
+    npaScaled = libMdlUtils.doClaims_stdScaler(pdfFeatEng, blnIsTrain)
+    pdfScaled = libMdlUtils.doClaims_stdScaler_toPdf(npaScaled)
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+
+    strParamTitle = "Std Scaled Claims"
+    return libMain.get_jinja2Templ(request, pdfScaled, strParamTitle, lngNumRecords, blnIsTrain, blnIsSample)
+
+
+
+@rteClaims.get('/doStdScaling/train', response_class = HTMLResponse)
+def claims_stdScalingTrain(request: Request, response: Response):
+    return claims_stdScaling(request, response, True)
+
+
+
+@rteClaims.get('/doStdScaling/test', response_class = HTMLResponse)
+def claims_stdScalingTest(request: Request, response: Response):
+    return claims_stdScaling(request, response, False)
+
+
+
+@rteClaims.get('/doFeatEng/', response_class = HTMLResponse)
+def claims_doFeatEng(request: Request, response: Response, blnIsTrain=False):
+    pdfClaims = libClaims.load_claims(blnIsTrain)
+    pdfFeatEng_claims = libClaims.do_featEng(pdfClaims, blnIsTrain)
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+
+    strParamTitle = "Feature Engineered Claims"
+
+    return libMain.get_jinja2Templ(request, pdfFeatEng_claims, strParamTitle, 
+                                    lngNumRecords, blnIsTrain, True)
+
+
+
+@rteClaims.get('/predict/kmeans', response_class = HTMLResponse)
+def predict_kmeans(request: Request, response: Response):
+    
+    #--- load test data, perform featEng, stdScaling, and fit to Kmeans args
+    pdfClaims = libClaims.load_claims(False)
+    print("TRACE: claims.predict.kmeans getting prediction ...")
+    pdfResults = libClaims.get_kmeansPredict(pdfClaims)
+    print("TRACE: claims.predict.kmeans prepping response ...")
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = False
+    strParamTitle = "Predictions (KMeans Clusters)"
+
+    return libMain.get_jinja2Templ(request, pdfResults, strParamTitle, 
+                                    lngNumRecords, False, blnIsSample)
+
+
+
+@rteClaims.get('/fit/kmeans', response_class = HTMLResponse)
+def fit_kmeans(request: Request, response: Response):
+    
+    #--- load test data, perform featEng, stdScaling, and fit to Kmeans args
+    pdfClaims = libClaims.load_claims(False)
+    mdlKMeans = libClaims.get_kmeansFit(pdfClaims)
+
+    #--- inspect KMeans data;  clusters, centers, sizes
+    #lstCenters = mdlKMeans.cluster_centers_
+    lstIdx = range(len(mdlKMeans.cluster_centers_))
+    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit_kmeans)  lstIdx: ", lstIdx)
+
+    lstSize = [sum(mdlKMeans.labels_ == idx) for idx,_ in enumerate(lstIdx)]
+    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit_kmeans)  lstSize: ", lstSize)
+
+    return

routes/qa/rte_providers.py

@@ -0,0 +1,188 @@
+from fastapi import APIRouter, Request, Response
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+import pandas as pd
+
+import main as libMain
+from lib import utils as libUtils, claims as libClaims, providers as libProviders
+from lib.models import mdl_utils as libMdlUtils
+
+
+
+m_kstrFile = __file__
+m_blnTraceOn = True
+
+m_kstrPath_templ = libUtils.pth_templ
+m_templRef = Jinja2Templates(directory=str(m_kstrPath_templ))
+
+
+rteProv = APIRouter()
+
+
+
+#--- get claims data
+def providers_loadData(request: Request, response: Response, blnIsTrain=False, blnIsSample=False):
+ 
+    pdfProviders = libProviders.load_providers(blnIsTrain)
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    strParamTitle = "Providers"
+
+    return libMain.get_jinja2Templ(request, pdfProviders, strParamTitle, lngNumRecords, blnIsTrain, blnIsSample)
+
+
+
+@rteProv.get('/data/train/', response_class = HTMLResponse)
+def providers_loadTrainData(request: Request, response: Response, blnIsSample=False):
+    return providers_loadData(request, response, True, blnIsSample)
+
+
+
+@rteProv.get('/data/train/sample', response_class = HTMLResponse)
+def providers_loadTrainSample(request: Request, response: Response):
+    return providers_loadTrainData(request, response, True)
+
+
+
+@rteProv.get('/data/test/', response_class = HTMLResponse)
+def providers_loadTestData(request: Request, response: Response, blnIsSample=False):
+    return providers_loadData(request, response, False, blnIsSample)
+
+
+
+@rteProv.get('/data/test/sample', response_class = HTMLResponse)
+def providers_loadTestSample(request: Request, response: Response):
+    return providers_loadTestData(request, response, True)
+
+
+
+@rteProv.get('/doFeatEng/', response_class = HTMLResponse)
+def providers_featEng(request: Request, response: Response, blnIsTrain=False):
+    pdfClaims = libClaims.load_claims(blnIsTrain)
+    pdfFeatEng_claims = libClaims.do_featEng(pdfClaims, blnIsTrain)
+    pdfFeatEng_providers = libProviders.do_featEng(pdfFeatEng_claims)
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+
+    strParamTitle = "Feature Engineered Claims Grouped by Provider"
+
+    return libMain.get_jinja2Templ(request, pdfFeatEng_providers, strParamTitle, 
+                                    lngNumRecords, blnIsTrain, True)
+
+
+
+@rteProv.get('/doFeatEng/train', response_class = HTMLResponse)
+def providers_featEngTrain(request: Request, response: Response):
+    return providers_featEng(request, response, True)
+
+
+
+@rteProv.get('/doFeatEng/test', response_class = HTMLResponse)
+def providers_featEngTest(request: Request, response: Response):
+    return providers_featEng(request, response, False)
+
+
+
+@rteProv.get('/doStdScaling/', response_class = HTMLResponse)
+def providers_stdScaling(request: Request, response: Response, blnIsTrain=False):
+    pdfClaims = libClaims.load_claims(blnIsTrain)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims, blnIsTrain)
+    npaScaled = libMdlUtils.doProviders_stdScaler(pdfFeatEng, blnIsTrain)
+    pdfScaled = libMdlUtils.doProviders_stdScaler_toPdf(npaScaled)
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+
+    strParamTitle = "Std Scaled Claims Grouped by Provider"
+    return libMain.get_jinja2Templ(request, pdfScaled, strParamTitle, 
+                                lngNumRecords, blnIsTrain, blnIsSample)
+
+
+
+@rteProv.get('/doStdScaling/train', response_class = HTMLResponse)
+def providers_stdScalingTrain(request: Request, response: Response):
+    return providers_stdScaling(request, response, True)
+
+
+
+@rteProv.get('/doStdScaling/test', response_class = HTMLResponse)
+def providers_stdScalingTest(request: Request, response: Response):
+    return providers_stdScaling(request, response, False)
+
+
+
+@rteProv.get('/predict/superv', response_class = HTMLResponse)
+@rteProv.get('/predict/xgb', response_class = HTMLResponse)
+def predict_supervised_xgb(request: Request, response: Response):
+    
+    #--- load test data
+    #--- filter to only those rows that are flagged with an anomaly
+    pdfClaims = libClaims.load_claims(False)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    pdfResults = libProviders.get_xgbPredict(pdfFeatEng)
+    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+    strParamTitle = "Provider Predictions (Gradient Boosting Classifier)"
+
+    return libMain.get_jinja2Templ(request, pdfResults, strParamTitle, 
+                                lngNumRecords, False, blnIsSample)
+
+
+
+@rteProv.get('/predict/logr', response_class = HTMLResponse)
+def predict_supervised_logr(request: Request, response: Response):
+    
+    #--- load test data
+    #--- filter to only those rows that are flagged with an anomaly
+    pdfClaims = libClaims.load_claims(False)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    pdfResults = libProviders.get_logrPredict(pdfFeatEng)
+    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+    strParamTitle = "Provider Predictions (Logistic Regression)"
+
+    return libMain.get_jinja2Templ(request, pdfResults, strParamTitle, 
+                            lngNumRecords, False, blnIsSample)
+
+
+
+@rteProv.get('/predict/svm', response_class = HTMLResponse)
+def predict_supervised_svm(request: Request, response: Response):
+    
+    #--- load test data
+    #--- filter to only those rows that are flagged with an anomaly
+    pdfClaims = libClaims.load_claims(False)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    pdfResults = libProviders.get_svmPredict(pdfFeatEng)
+    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+    strParamTitle = "Provider Predictions (Support Vector Machines)"
+
+    return libMain.get_jinja2Templ(request, pdfResults, strParamTitle, 
+                            lngNumRecords, False, blnIsSample)
+
+
+
+@rteProv.get('/predict/enc', response_class = HTMLResponse)
+def predict_kerasSeq(request: Request, response: Response):
+    
+    #--- load test data
+    #--- filter to only those rows that are flagged with an anomaly
+    pdfClaims = libClaims.load_claims(False)
+    pdfFeatEng = libClaims.do_featEng(pdfClaims)
+    pdfResults = libProviders.get_encPredict(pdfFeatEng)
+    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
+
+    lngNumRecords = libUtils.m_klngMaxRecords
+    blnIsSample = True
+    strParamTitle = "Claims Predictions (Transformer/Encoder - Keras Sequential)"
+
+    return libMain.get_jinja2Templ(request, pdfResults, strParamTitle, 
+                                lngNumRecords, False, blnIsSample)

routes/qa/rte_qa.py

@@ -0,0 +1,17 @@
+from fastapi import APIRouter
+
+
+m_kstrFile = __file__
+m_blnTraceOn = True
+
+
+rteQa = APIRouter()
+
+
+@rteQa.get('/')
+@rteQa.get('/verif')
+@rteQa.get('/valid')
+def qa_entry():
+    return {
+        "message": "qa routing - For verification, validation"
+    }

templ/templ_results.html

@@ -0,0 +1,4 @@
+<!DOCTYPE html>
+<html>
+    <body>{{ dataframe | safe }}</body>
+</html>

templ/templ_showDataframe.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Fourthbrain Capstone:  Healthcare Anomalies</title>
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-+0n0xVW2eSR5OomGNYDnhzAbDsOXxcvSN1TPprVMTNDbiYZCxYbOOl7+AMvyTG2x" crossorigin="anonymous">
+</head>
+<body>
+
+    <h2>{{ paramTitle }}:</h2>
+
+    <!-- Mark data as safe, otherwise it will be rendered as a string -->
+    {{ paramDataframe | safe }}
+</body>
+</html>

uix/images/image1.jpg:Zone.Identifier

@@ -0,0 +1,3 @@
+[ZoneTransfer]
+LastWriterPackageFamilyName=Microsoft.Windows.Photos_8wekyb3d8bbwe
+ZoneId=3

uix/lit_packages.py

@@ -0,0 +1,36 @@
+import importlib
+
+
+#--- return a list of streamlit packages/pages to render
+def packages():
+        #--- 
+        ary_pkg = []
+        ary_pkg.extend(['lit_continentData',
+                        'lit_countryData'
+                        ])
+        '''
+        ary_pkg.extend(['lit_claimAnalysis',
+                        'lit_claimAnomalies'
+                        ])
+        '''
+        return ary_pkg
+
+
+
+def get_aryPkgDescr():
+        #--- load list of pages to display
+        aryDescr = []
+        aryPkgs = []
+
+        aryModules = packages()
+        for modname in aryModules:
+                m = importlib.import_module('.'+ modname,'uix')
+                aryPkgs.append(m)
+
+                #--- use the module description attribute if it exists 
+                #--- otherwise use the module name
+                try:
+                        aryDescr.append(m.description)
+                except:
+                        aryDescr.append(modname)
+        return [aryDescr, aryPkgs]

uix/lit_sidebar.py

@@ -0,0 +1,99 @@
+import streamlit as st
+import importlib
+from uix import lit_packages
+
+from uix.pages import lit_home, lit_about
+from uix.pages import lit_anom_superv, lit_anom_unsuperv
+
+
+
+#--- alt define sidebar pages
+m_aryPages = {
+    "Home":                         lit_home,                      #--- TODO:  update
+    #"Provider Analysis":           lit_providerAnalysis,
+    #"Claims Analysis":             lit_claimAnalysis,
+    "Provider Anoms - Supervised":  lit_anom_superv,
+    "Claim Anoms - UnSupervised":   lit_anom_unsuperv,
+    #"MLE Model Performance":       lit_about,          #--- TODO:  update
+    "About":                        lit_about
+}
+
+
+#--- define module-level vars
+m_aryModNames = lit_packages.packages()
+m_aryDescr = [] 
+m_aryMods = []
+
+def init():
+    #--- upper panel
+    with st.sidebar:
+        kstrUrl_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAf4AAABjCAMAAABNPpI+AAAAxlBMVEX////rAIvrAIrqAIfqAITqAIP/9/zsAI/1iMfqAIHwZqrtDZfyb7Pzbbf+8vr3p9L4rtf6zeX95vLtI5X0dr7wX6vyWrH/+/72mcv1jsXvNaH61ub5uNz4q9n6x+X1pcruSp794vPwT6v96/X92/D4tNntMJn81ez6weLzebrwVKj7zef95fPvQaLzi7z82e/3otDwRKjuKZv2lcryYbP1mMb0gMPzg7zuPZzuTqL5wt70nMbygrfvWqXsJ5L2r9DxdrH3ttMtxwOlAAAZMklEQVR4nO1de2OaPheGJIj+VNRSi1rninhFWa06q93mtu//pd6cJNyDYl/b2c3nj80KQsKTnJxbDooih9N60tFy2rAyjoc4fsYVHw31EUGqijB6dg6eN259/fX1k/FOrbrifVDDlHwAws0D3DpTFVMgc/F+TbvizTFWKfuYaDAItOfM0+Ye4aMEkWLhHZt3xZuiUMUqQuvG/YoOA6QPMk4zmhjEA2b/tt61hVe8IW50pOI6fLJHlP/i3DEoUipeHXg3m9+bMEjIYR3hio+DsabiPv84pMyao12TovKdor2iWAy7w+Fk8IRVvBwXFKNG+SftP9vmK86GGlHJC/9owcxGIOGjQAC2MMz5D6im8OMPNviKc6KnBfQroAQiAVDy1ACgHQq10NkitL1K/78EDl36p/zjDIje7/ch6SgQBlTi/xS/+HGl/+/BEFS/BnwyXLq+N+nyblPMuhSfWhSrLxTfdYTFgk9nP27+yRZfcTZYKx0EvrkwlJnLhP3uRnrihiCT+4S6OkKj4Xs28oo3wrhMEHh7ESaEYPikYrUmc+w/mgh3bEspdNl4IZ3x1f3/MTGf1IaP8MFq6dSax/pUxUzXQ/tbHcbAVLKyW0VCLb/p1ypTD9mvruv/B8Sguqfz13Rnit3HYOi5A2Xs7RmrW6exBFEwmkl+eIeZ1w9swD38i0e1q/P3o6GmY7DiEdamJpv6CybE7UltjZE5VoxnWABwK81sC4u4EG7N2yA2kOa+pM664pLRBf8NuHIo8+C99cJ5XidI78L/sAAgNynaLZ39kqA7SAsYPxG2Atxew78fCLYJ3rvmL1dnbh2zGDnmlFT8jX2o0NGB9WFct2vTQaHfPM79v7s7pjXqi6sK8GGwIJTDGTXthyAF9r3YQboiTBnlhRbV/5E6jfI/34buIQ5npTMB4k7evt1XnAVPCOlj9mlIrXczHt19wsgTstz2wN3nNcKDX+mAMB/jV7PXKqgPeM1cBc7N+P7mqgxeMkx/givODvGlPsSUTmbf5+OsmAugzk6e92Yzumrgdep63b4GKoC5Muz2kmiaXr2KggsGEss7RYVO3Frs4AtRtfvwL5OlgTjKYL01TRNYHqcvaPAVAD+MWMYY1QXaf5c7CHpjXWSXjNWu2T5N2oaz33Dp7I9P1YaOSCSPxwFpgE3uEwL219KnMK8wI9CPDiKSnS/2AbEpAdD8+JnvDWekUTtsJHfRZwBse96VHl371U+xg4MlwnfRL2o69wNz+gPFIAGr19HYzCelEhgDROYz+qgogrMDlU96yO+DhQazjRSPnxliTOks25ZizZjlx7z4AYwmRp3Y6Y7LJrT548cWJIGXJQRbYE+OFgWlPqKnHQoHFo7isgTtKfRLOvOGLeszL1yCsCOwIK/TnH6b6oHnPqL9g8MvPsMhwRP3Z5ZiT+EH9YzLgh7JzYiBidDoMeM0iq+VY8i6x5/BKfTfpjvz9VutK9GYzoHX0K8UPIyE5x6Z4P0Frd0/2KJfxyW342JUZsetCiQDZPj46EpCRPb/giD9gCv4P4IPQ9uc1J+3xin0l2X9Qbq5XU/eQKRVGf1+mmZeFNjEh5W6PVgxpY0su6J1Q7rGxyefXQ60wTGVG2VbftEXVdXEIYfqAYfox+ph4A9MP5J1CBwoRF9lPLnX44XvzOoePzMG6x7415Zg0o/7JRgAqssb90gvt4qdPNdRYAuC0/dekeJFRZrvMNKQ3pOfBfj36BdxstHZF7U2oXiFmUWbSfzBOHGZ2Y5v6ReFAR2q+tMwkPA3jTU9VYyvG5y9CaQXpoK1MXo4EAUQ9KNMkD9Df6FfLusSEX06/UGitMiX9QdA6fbcK4Dx6dNr4i20WXrwO+cb+G1UMqrZFaYPYuzy9X/c8hDIBo+f2KL0L+UizGmCecA8xA0Tkan0JA5Ov17OhH6SJXM2OPShlM5BPzL7Ah3am5JGcOARuZC9USXan8iwsasq89uYfkOxObOcuqdiLMQxFQdWHTJCn6UD2KbqJIiG4v1NEbwJh8wdRj9uOtaFGX4toqpnoR/fKZYA9GbccvfiuSKUtYHuXVGg9C9jKnzXJdwORISOVtqFXUUnPNOfRfW9r5+rLMtHuqhPYNxUQMPRNDoO9IPKiKD/4rIEXHw++hOw21zZlsVM/gDYrq748y8UhRdoaNc9tomTO/BxR/zJBIFcK+uaGOt1q8h+Q+3Iw2buhdI/h5jGG9HPvOls+ptnV/9fgRml/yn5/AdhRK8qZD7ynimV812gvyBPItZXVOsbAeWP7T1CR/XbC6W/Dv17M/qVHt8whWqSY++NCVFxNUlkI/TW3YxAFcR3Ex4cMHZgGhANFoVV4leK1cKIbMWEb+Lj2z8ulP5naFZJcuA89IPPDI59kxx7b1AtB0+TA31IKf4lPn/BKm6FumETI726GkOyl5pY140pZb/qn3qL0dGtn5dJvwMjXtUKvkLqOP7zOQ/99PEy+isXkA0jod9YeFT2V8RflH4SHnMe+JY+y4NwQaz9Dh0ZEUFSxGh7LDJ6Iv32rF6vfavXJ+dVmgs9etUQn78IdqbMUT+t9Dt+nlOC/jltT73e7UkX8QP090x0kH5jUq99pt1MPL5BbwL3m8xeMV1s2sfPtfpwlrgn0P8l+kWhuOUBXSHtXYxQeHRODz7Ah0c9ke1n7yj7m3AgtQgaHaPpFPqdlTsyoe4MUvX9wyY1sh7dB4BsQe2xI26gho7hz63wWdL+mmp8G7uweMO/fb81p19n9BcWrD2qqpuj3Sbd0wP0D0YJ+ofQID+COqhudW5mh04Py1hUH0amzu+3dVOZ90af9TGstNDd0j//46K4UHNHOtfgze06JrNvcbhZl8JugZEHNxc7uFc41gfQCh749cGPFdZ1ui/TVSHqyugSZB6L9eenvzElBPt+M7bRuJ8wKhomeNbSCglrKkS2wtY0IGelxOgvrIRRewDCh6VEZ79Dn5NoD9v3rFWTfT1OfygpazjwMzkb5PeT+AkYdrePSOA35GGDRNk1owzHyW3wRV3zu2zUypp/SdbUTkQjp/Sj3bTNM33mzyO2w68/JWDZ2dRKhSzvSNBmbAYbequEPpjeavqr2OMPX40p+jMtmT+URm76W2YqOoD1uIe7wSSq1JnW5SSG9IPpRaC1TgUf4R5+GUgxTr/nsOTXRHPMhNw5LvxDoVtjygasIeA08y+pcceKU9ziVCtRYvgb/HYR+gnIqQkPvyeaioIUPMfllry2nBQGU8JmSXWsWPA15m4fFFUNJtSyq/KPVoeOHBaxJZ02oQZf3A1klVT12M7fnPTPXRLam+En0onOgFPph7DE3CNHyadnun4DBf3GRMV+a8Lm4HiNmwP0d7nlH44XRj+hXM2iw7zEb7sIOg9zN7ghjoVcpPSr5DMdapKmkiqn1FpjLr3o1Sqw4YvoUxhV1lR8jxLuCaDf9/c0RkhcFdEVopP08Ggq/qkcRj76HZeIB0wNznIZBZ5z3I/wfzL9a8W6I/y6sKM5hHheRBMoBQYsox9XByMxZ7Ryuaxpvhs37uE8QD8zLJEa5sFw+heKw7gKCurwgwbvGJXaernT75RLfu+jjhc5/fj5Zon5b0u0qThoKuardhuEvLndqnxjN9oX+QM1YA/X3tT1Ef0yupz+pqqC//cjHNxvR8xBvEypsWWEP8s6H0Eu+gsV4XlS3bYDhtjNp74Y0jiS3XIy/XewVQkuu78rticRLNjFSXdmCESsGbhQ02PbId3iHFpjzKp+a5bx7mfRb/NDEacvo59KD14uT91WixT9J3F0Q7VzbO6KY26H3rT3gsTIciOnH3lP0FTVaw/YTxsbnoKt8hxPB1imsl7pMm/uaOFP9K5GfzWfTXoGLI6RUf2bhBoJHGrNFaNO+ZcU91uitEMhgVz0t0VUOLp/aCzW7IjtcSr9qDxm+xR39eTt71naZKbbh60/qBlZ6mx/gEaX/0z6oWwKTP6ItOT0Vxcs1pLaHGEj4q2isvWGl96ICj85/aKpkQs6U74GML5qiNVvgStC3DdUXu8Cfd+h9pwZPqCv2NdIFJuKC/6woQRcOdXNPk67ExPIQ/+cE03i17JaPAgVJpOcTL8JrgvSTt9c0J/l9GWtiYfrDZFrVYl8m0W/3eTCJaopMPrREiop6vXUMytsuolIPnPF0l+Ey3IG/UzKxHpoPfOmgj7zhT6AcdAzLdDw5/tQ34dCHk0r8mNN/GJF0E7YlbT36clyi9PBhATy0C+ebDX5/YJw8e837VT6VVafThaVOE5/qjVzjxMYSWuV0G859ktVY3OfxKLlNWHRqcdiZEGPeF53aHdn0p9K/yvsuPC5B/oDl06DRK5WQxFrp6Yh1T8C7mr/uVRx4Bn8RgdFyg/TxolIcho56B9zViUOxA6fRb5kO5l+eN7SetRH6UejlJ9vw49EtkHydI9Oy0f7+5df7pK5cxDWV7Gr13yHhplzV1xBT6gPWfT7mzgjaLEjUL4PZr949kMSzn5QtsJVlVoBCIv+FvpU9RMtp7ObLxxMJqBl8T5O4096m4VzcPUX9B9aIngZCSxx5onNCa748xX0R55WFEfpl/iWbD4dI80sBz4hFFbE41bSvpoYPj79sm7KcceGUahrZtEf9+hycLFJ7Teqa/h3pCpisJTY4NAN9T3Hw9gTYv4JobJ4LqvArceWImqWeV8mwSJldUEeErVyKKrN6Efbb59T8Ce71eFLvISLAltEkS8uT6dfeG9TOEa/NFEHCeU9QEaqJzJX3dQjEfSjZfJAJtr56Jd63jpIrF+2iZAJypMFvt3Au0ElAVIj51MlT3SM3iWg38HMAaYwH5FI79C3VZ7FXpgi/qjI6MBqxnP9knVjoWiIL4gcPlSl5vOCkyicC6fTL3UQKznol22mKOekX1Xd1Sx5aUE/yZ+n/ZvZdXrwdxb9I8lv+1jQT2ewirTbn5/hG9z0B/USJeRijVJUF3fBfdF4G07T2y+tJWV/29SJcDCo1d8D447Qg8yDgrO2AyjZid7h3hCu4GnS4MGA+06FEnW66qc3JOcqx+mX5jk9sWca0bMy6aePqLyKb30Sql8nf6ruC58Wwd8Z9Eszypgyzcaw4fo0gdopNJo5+ILic5aa2cz5Z8AuL/5cIMLHUj9YbvjQGNefCOEyAI9c5pWuN6ZUbUwr7QGO03/L3WDyyDFzf/ipaqfTv8tQOY/RT2QKI1uMo1KK049LUWgiTISwGYvZcfozcmelyEu/LFO6HdDPhDQr7QV79nlipkE7ifrxgWh5XM8zSr6A60KET8Qi0J5LrcKiueWrAPJ9UjNYXjLF/3H6uYtkKV+kuf4j8pRPph9npaAfo1+TtUZOf3zVciaLyoMuFkU3MqTF7D+q+FmFwo3jzG17/hnnol+6nETopwt9hRK5nDYaEMHRu4/PUOgJuYn51uBbNxqaCAR3YQTX562ODvM7jL3Zw6muRZ8EeFAzU/7E2k+S0IIg4wPj1JNLRS4aRA2Sk+knGUv/ceH/WvoBTnfNQzjEDYUPd/scDpDak8W350rV87ylrpeF9hIczaBfk61vMfoVS7yv4wa8YKopgov7xLipUwnRAKWQ/IainqqI8BUM0AA7ETFqGb1b6J248Vw7kNTGNX+v3k1B8G0kEyNi2EQn4+n0f5KcCjhCv9xgyEs/fUA1NSnrOf1m5nY4yy5u9zoK3qfgR++O0y+bN3H6w0tUSKQuR9IlMiVo1KY2A/41m9MlHXu+fAAHcUK7o18FmwGP0n/I7rcP0s8dGO9OvzTXLz/9LJWSPeFgUTxMf6HWKWEkS0o5Tr9Mvcmgn8fWkB/m1+Mjx1kinvCPzS1CJFQOqA2S1DCmONCPxuTAmnbU6zc/SH83KuA+EP3KOuHIPkj/zI1kF/iWMX4D+qFZSF0NlEEFpY2GKcsAwZiF+CPHxpAVHj/1Bfsve4FUwX1mgYej9B+e/fUPOvuFNyOMkxyif4FEjBbTeef++jUtFouL+iaf6ncS/Ybm+x0h3QNtY7+FHf24+nvRh4o91cizKfxH2xf3g9149ByvaymzprQCnI+j9DsH6S/mXPvF070c+rnjTcU+3wfoF2/VRKQ8/d0YBE8qp+F3Ev2w42fNn7S9RfAml6d+v7rZtIb1XoFOY2YXWi0Uj2yybNCvsQvZOxa53u+p2MIZrlXA8ZAP0/yzzsir+rUujn4RdfVtj2z6ea/gDRvxGfAW9NOfBAk6P7gPj680zBYLXJp31FCMzvZHnJifjsfcSDyJLFufzUP/7rjhp39A+lecfl9lyqTfuuMn9pNRht9vQH8v1NKtH4jX64+YGf61XkjCX1pGsb3+N1ABarnkQ6dzaKvHcfrX2Q9cUXiQ/bjbZ4Uvjf6vnFU/RJBJv8MdI7okmn5++i0SlOubmQi7lWbTG41MXVeZLeBrKramohj9kCoSJvXO6dzHo4EyWX1/Xv2/O3xFcFpeRoY7fe+44Mmm3/p1cfS3c9Lf5t1Pm05vQT/EgrhAKrhILdmWYcwH9rhB8dIM03ygXFeM1nF0KwikjuNlvqSV4/Q/llj/pWkZM2Y++xHtWSb9jntx9N/mFP5c+OmpoJlVfQv62c6t/vAedlUkXOI/6ZEdm2cOCIbYYuxQKezvA7R3UPItZ93L4/QXRNKM7BhfQP3NBFBjPh5y9cEjg+FenZz0S46ci35L+Gx9b3gm/Vk+byuv0/ck+pU6lOskGuzzStwTHDCoObaUCfCbmGP0giK44MDRDE0tjeP0sw0HasqtCBAZs35SmsONhF/pScuDxqfOftmTOxf9E3YHlQT5dIfp76ee55hf4Nz0w7tb2FY5lMrSHIK9b263ICCSSQTwMr+O+7C7nYDW5+YuW5Ej1+8lMzrX5QPD34NREEZCmpslehX9EovlXPS7QqPz/z559q/fxOtHMW81R6ZXkQQKV6qw5dA2aYaAd4blsxFgP3/SQg76xRxP1wbl2Y4qCbQMUTQhdWKNz5QT6OdRfUlA8Ez0F7klFZbhy6SfD2kzecOumjfkcyr99Er2YC7zslkvusayFeNOH8CEjQwc3RieC3kSvYdM+cOj5KASnpMwv2ZFEl9wcL/xKfSLXTiS2iTnob8oiruEZTEz6ecaHklEgh3RpTeh/wB6xU0RFveEXg8aI9Ldyk5HMhs1G7l2+TxxnuM5g05FLOihIHL4LNeKscH5WPbjJblVP2EpSDaonoF+Z7IUWzYjClQm/UPWJ1SOTUdb97Nk3pl+ADW3kp26hUDxxFCcLp3/p9SUz0X/2Pd7RvKjJh5/ArG9Mvw7pEZeKHqzYq8nPM3rJwQL2qboOJ3+/o0dYjJcfPf8/bo48kKEbLeP2JJ1F97SWUG9/LcT/kcAdf5RvKFU8RNr81BNRf8OId8O36HIy0bmunvjOE53MxIh0Likb6j+edW64TjG/eKOFVHEUxhAJ9A/E2aVensPqTC9jZ/beTL9EDcJISps8KYvI6tZts+fj0R4yZ7Ndpt216zyuv6JaQV/gH4o6xFXxAehzx9Kvu/zS/+c+/vr/q4capNqWAvKfCS0TLF9zU9Z0MRGcLy2RqfR7yvWbE91uUQ0P7n5FfTHaxQL7lWkraNPKZv+QHNhTSmLZFq9zhXdP0G/0sHxl771IhG/7/iUcoV5q3sMdUkRDkSSO0itDkmdhnCnwDLbT6Hf9hUGXr4gEDKvoF8GhNX4W5cOBHwb0eIOYtRDYYjiH6Mf6I4mb9s4KA5WoCvDPn/hqdzFXZymn/ESkr9NZ5Bam+RpWN0UeIdPoV9pjKLXOSv9COPtVLbJKyPdoztKdmkEhsCE/Cn6DSjaHH0EsPZzcdAwEfbyX4m9zYPkKe1U6D1pYZ4jiMK2bI2xJh0iiqezzCjNe4Fx+VKi9wnpn7GqXaUD9Ct2n4R3056E8N+A5StPYOizrkS3eaTf5QFx89Ly7ndKPC7Y4Yw3H9jrSFPossaLmlnQJS04ydDZy08i9JdYH6WpnqwX+TeUJTFM1HSD8lDlG0uxBmV0wjZFRWlVAat85Q3tors1TV01zdGumr0danzrn7Z1NyLNzFnT29wGT91mt10frjvWqO7EZXZFf6kbsh/eyp6p6Er4xW01hdaqlayrx9Hj182qgzfY7KComw5dCl62s4GfBKcU+O3CET1bsz7KZhbvxetL5Rseiu1xBv0Em9PPUx2cfm9Yg93ozbrD3uxYucB5bzahp/2/r9uzxj24zEW8lHrQ6w27vcZl1D+tkVh6lwOpa/yVUPjQjs4r/hIso68AgaQEJCJFyd1BV/yNqOOIw80G1a+2pmZp9eUypNMVb4s5Xf19A89qYmmWxRV/L1ZU2YcMLNuBiLz/KuAr/hEUKOfeJ3e033tU7ZfkIl7xV6NFVFFlGmoDXtablq94c/DKgohtB4/s77/i3wCUclSR+V/lgQ4AWV7cFX8zGiD0+2NLuXnGYXG9K/4RMD8/X/KfcbQ4+RX/Ajoo2HtjY1le3BV/M5Yo3HmHMorjXvHX4gkH1v4jRodf1XvFXwcI84iEkRUV/lev378FVvYZXtlpLKj1fxFvo77i/cDSanHny1eXBKleV/w7gGI//GXuWL8qfv8erIrKygGT0VXv+xdhjasdVX9aXNW+D4n/AeKsA1opSbg6AAAAAElFTkSuQmCC"
+        st.sidebar.image(kstrUrl_image, width=200)
+        st.sidebar.markdown('Visualize Provider and Claims anomalies.') 
+ 
+
+    #--- init checkboxes
+    strKey = st.sidebar.radio("Go to", list(m_aryPages.keys()))
+    pagSel = m_aryPages[strKey]
+    writePage(pagSel)
+
+
+
+def init_selectBox():
+    #--- init module array of page names, and descr
+    init_modDescrAry()
+
+    # Display the sidebar with a menu of apps
+    kstrMsg = """
+            __Claims Anomaly Views__
+            """
+    with st.sidebar:
+        st.markdown('---')
+        st.markdown(kstrMsg)
+        page = st.selectbox('Select:', m_aryModNames, format_func=fmt_modName) 
+
+    #--- display sidebar footer
+    with st.sidebar:
+        st.markdown('---')
+        st.write('Developed by Chavarria, McKone, Sharma')
+        st.write('Contact at iain.mckone@gmail.com')
+    
+    # Run the chosen app
+    m_aryMods[m_aryModNames.index(page)].run()
+
+
+
+def init_modDescrAry():
+    #--- init global array of page names, and descr
+    #--- note:  you need to specify global scope for fxns to access module-level variables 
+    global m_aryMods
+    global m_aryDescr
+
+    m_aryMods = []
+    m_aryDescr = []
+    for modName in m_aryModNames:
+        modTemp = importlib.import_module('.'+modName,'uix')
+        m_aryMods.append(modTemp)
+
+        #--- If the module has a description attribute use that in the 
+        #--- select box otherwise use the module name
+        try:
+            m_aryDescr.append(modTemp.description)
+        except:
+            m_aryDescr.append(modName)
+
+
+
+#--- display the app descriptions instead of the module names in the selctbox
+def fmt_modName(strName):
+    global m_aryModNames
+    global m_aryDescr
+    return m_aryDescr[m_aryModNames.index(strName)]
+
+
+
+def writePage(uixFile):  
+    #--- writes out the page for the selected combo
+  
+    # _reload_module(page)
+    uixFile.run()

uix/pages/lit_about.py

@@ -0,0 +1,24 @@
+#--- about page
+import streamlit as st
+
+description = "About"
+def run():
+
+    print("\nINFO (lit_about.run)  loading ", description, " page ...") 
+
+    #--- 
+    #st.experimental_memo.clear()            #--- try to clear cache each time this page is hit
+    #st.cache_data.clear()
+
+    st.markdown('### About')
+    st.markdown('### MLE10 Capstone:  Healthcare Anomaly Detection')
+    st.markdown('#### Team:  McKone, Sharma, Chavarria, Lederer')
+
+    st.markdown('Kaggle Claims Data:')
+    st.markdown('https://www.kaggle.com/code/rohitrox/medical-provider-fraud-detection/data')
+    st.markdown(
+        """
+            About page
+        """,
+            unsafe_allow_html=True,
+        )

uix/pages/lit_anom_superv.py

@@ -0,0 +1,368 @@
+#--- anomaly detection - supervised page
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+
+import lib.claims as libClaims
+import lib.providers as libProviders
+import lib.utils as libUtils
+
+import sys
+
+description = "Anomaly Detection - Supervised"
+m_kblnTraceOn = True                                  #--- enable/disable module level tracing
+
+def run():
+    #--- note:  in python, you need to specify global scope for fxns to access module-level variables 
+    global m_kbln_TraceOn
+    print("\nINFO (litAnomSuperv.run)  loading ", description, " page ...") 
+
+
+    #--- page settings
+    if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  Initialize Page Settings ...")
+    st.header("Provider Anomalies - Supervised Approach (XG Boost)")
+
+    #--- provide file drag/drop capability
+    m_blnDisableDragDrop = False
+    if(not m_blnDisableDragDrop): 
+        #btnSave = st.button("Save")
+        pklDropped = st.file_uploader("Upload a Claims Dataset", type=["pkl"])
+        m_blnDisableDragDrop = (pklDropped is None)
+
+
+    #if (True):
+        try:
+
+            #--- show:  raw claims data analysis
+            if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  load raw claims data ...")
+            if (m_blnDisableDragDrop):
+                pdfClaims = libClaims.load_claims(False)
+            else:
+                pdfClaims = pd.read_pickle(pklDropped)
+
+            #--- get supervised predictions
+            if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  doFeatEng (claims) ...")
+            pdfFeatEng = libClaims.do_featEng(pdfClaims)
+
+            if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  perform xgb prediction ...")
+            pdfPred = libProviders.get_xgbPredict(pdfFeatEng)
+
+            if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  get sample ...")
+            lngSampleSize = min(50, len(pdfPred.index))            
+            pdfSample = pdfPred.sample(lngSampleSize)
+
+            #--- save a test file
+            #if (btnSave):
+            #btnSave_testFile(pdfClaims, pdfPred)
+
+        except TypeError as e:
+            print("ERROR (litAnomSuperv.run_typeError1):  ", e)
+
+        except:
+            e = sys.exc_info()
+            print("ERROR (litAnomSuperv.run_genError1):  ", e)  
+
+
+        try:
+            #--- save this file locally as a pkl
+            #btnSave_testFile(pdfClaims, pdfPred)
+            
+
+            #--- table sorted $insClaims reimbursed by provider
+            #--- display providers with predictions, sorted by InscClaimAmt Reimbursed
+            pdfTopClaims = pdfSample.sort_values(by=["InscClaimAmtReimbursed"], ascending=False)
+            if (m_kblnTraceOn):  print("TRACE (litAnomSuperv.run):  Show $claims reimbursed by provider ...")
+            st.markdown("(Top) Ins Reimbursed by Provider")
+            st.dataframe(pdfTopClaims)
+
+
+            #--- chart Top Insurance claims ($) by Provider")
+            chart_topInsClaimsByProvider(pdfSample)
+
+
+            #--- chart Top deductible amts ($) by Provider")
+            chart_topDeductiblePaidByProvider(pdfSample)
+
+
+            #--- chart Top IP Annual Reimbursement amts ($) by Provider")
+            chart_topIPAnnualReimbAmtByProvider(pdfSample)
+
+
+            #--- chart Top IP Annual Reimbursement amts ($) by Provider")
+            chart_topIPAnnualDeductAmtByProvider(pdfSample)
+
+
+            #--- chart Top IP Annual Reimbursement amts ($) by Provider")
+            chart_topOPAnnualReimbAmtByProvider(pdfSample)
+
+
+            #--- chart Top IP Annual Reimbursement amts ($) by Provider")
+            chart_topOPAnnualDeductAmtByProvider(pdfSample)
+
+
+        except TypeError as e:
+            print("ERROR (litAnomSuperv.run_typeError2):  ", e)
+
+        except:
+            e = sys.exc_info()
+            print("ERROR (litAnomSuperv.run_genError2):  ", e)  
+
+
+
+def chart_topOPAnnualReimbAmtByProvider(pdfSample):
+    pdfBar = pdfSample.sort_values(by=["OPAnnualReimbursementAmt"], ascending=False)
+    pdfAnoms = pdfBar[pdfBar['hasAnom?'] > 0] 
+
+    #--- chart 
+    fig = go.Figure(
+        layout=dict(
+            title="(Sample Anomalies) Top OP Reimb Paid ($) by Provider",
+            legend=dict(groupclick="toggleitem"),
+        )
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=pdfBar.Provider,
+            y=pdfBar.OPAnnualReimbursementAmt,
+            name="OP Reimb Paid",
+            marker_color="LightBlue",
+        )
+    )
+
+    
+    fig.add_trace(
+        go.Scatter(
+            x=pdfAnoms.Provider,
+            y=pdfAnoms.OPAnnualReimbursementAmt,
+            mode="markers",
+            marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+            name="Anomalies"
+        )) 
+
+    st.plotly_chart(fig, use_container_width=True)
+
+
+
+def chart_topOPAnnualDeductAmtByProvider(pdfSample):
+        pdfBar = pdfSample.sort_values(by=["OPAnnualDeductibleAmt"], ascending=False)
+        pdfAnoms = pdfBar[pdfBar['hasAnom?'] > 0] 
+
+        #--- chart 
+        fig = go.Figure(
+            layout=dict(
+                title="(Sample Anomalies) Top OP Deduct Amt ($) by Provider",
+                legend=dict(groupclick="toggleitem"),
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=pdfBar.Provider,
+                y=pdfBar.OPAnnualDeductibleAmt,
+                name="OP Deductible Paid",
+                marker_color="LightBlue",
+            )
+        )
+
+        
+        fig.add_trace(
+            go.Scatter(
+                x=pdfAnoms.Provider,
+                y=pdfAnoms.OPAnnualDeductibleAmt,
+                mode="markers",
+                marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+                name="Anomalies"
+            )) 
+
+        st.plotly_chart(fig, use_container_width=True)
+
+
+
+def chart_topIPAnnualReimbAmtByProvider(pdfSample):
+        pdfBar = pdfSample.sort_values(by=["IPAnnualReimbursementAmt"], ascending=False)
+        pdfAnoms = pdfBar[pdfBar['hasAnom?'] > 0] 
+
+        #--- chart 
+        fig = go.Figure(
+            layout=dict(
+                title="(Sample Anomalies) Top IP Reimb Paid ($) by Provider",
+                legend=dict(groupclick="toggleitem"),
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=pdfBar.Provider,
+                y=pdfBar.IPAnnualReimbursementAmt,
+                name="IP Reimb Paid",
+                marker_color="LightBlue",
+            )
+        )
+
+        
+        fig.add_trace(
+            go.Scatter(
+                x=pdfAnoms.Provider,
+                y=pdfAnoms.IPAnnualReimbursementAmt,
+                mode="markers",
+                marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+                name="Anomalies"
+            )) 
+
+        st.plotly_chart(fig, use_container_width=True)
+
+
+
+def chart_topIPAnnualDeductAmtByProvider(pdfSample):
+        pdfBar = pdfSample.sort_values(by=["IPAnnualDeductibleAmt"], ascending=False)
+        pdfAnoms = pdfBar[pdfBar['hasAnom?'] > 0] 
+
+        #--- chart 
+        fig = go.Figure(
+            layout=dict(
+                title="(Sample Anomalies) Top IP Deduct Amt ($) by Provider",
+                legend=dict(groupclick="toggleitem"),
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=pdfBar.Provider,
+                y=pdfBar.IPAnnualDeductibleAmt,
+                name="IP Deductible Paid",
+                marker_color="LightBlue",
+            )
+        )
+
+        
+        fig.add_trace(
+            go.Scatter(
+                x=pdfAnoms.Provider,
+                y=pdfAnoms.IPAnnualDeductibleAmt,
+                mode="markers",
+                marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+                name="Anomalies"
+            )) 
+
+        st.plotly_chart(fig, use_container_width=True)
+
+
+def chart_topDeductiblePaidByProvider(pdfSample):
+        pdfBar = pdfSample.sort_values(by=["DeductibleAmtPaid"], ascending=False)
+        pdfAnoms = pdfBar[pdfBar['hasAnom?'] > 0] 
+
+        #--- chart 
+        fig = go.Figure(
+            layout=dict(
+                title="(Sample Anomalies) Top Deductibles Paid ($) by Provider",
+                legend=dict(groupclick="toggleitem"),
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=pdfBar.Provider,
+                y=pdfBar.DeductibleAmtPaid,
+                name="Deductibles Paid",
+                marker_color="LightBlue",
+                #offsetgroup="anoms",
+                #legendgroup="anoms",
+                #legendgrouptitle_text="Anoms",
+            )
+        )
+
+        
+        fig.add_trace(
+            go.Scatter(
+                x=pdfAnoms.Provider,
+                y=pdfAnoms.DeductibleAmtPaid,
+                mode="markers",
+                marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+                #offsetgroup="anoms",
+                #legendgroup="anoms",
+                name="Anomalies"
+            )) 
+
+        st.plotly_chart(fig, use_container_width=True)
+
+
+def chart_topInsClaimsByProvider(pdfSample):
+        pdfTopClaims = pdfSample.sort_values(by=["InscClaimAmtReimbursed"], ascending=False)
+        pdfAnoms = pdfTopClaims[pdfTopClaims['hasAnom?'] > 0] 
+
+        #--- chart 
+        #st.markdown("(Sample Anomalies) Top Insurance claims ($) by Provider")
+        fig = go.Figure(
+            layout=dict(
+                #xaxis=dict(categoryorder="category descending"),
+                #yaxis=dict(range=[0, 7]),
+                #scattermode="group",
+                title="(Sample Anomalies) Top Insurance claims ($) by Provider",
+                legend=dict(groupclick="toggleitem"),
+            )
+        )
+
+        fig.add_trace(
+            go.Bar(
+                x=pdfTopClaims.Provider,
+                y=pdfTopClaims.InscClaimAmtReimbursed,
+                name="Ins Claims Reibursed",
+                marker_color="LightBlue",
+                #offsetgroup="anoms",
+                #legendgroup="anoms",
+                #legendgrouptitle_text="Anoms",
+            )
+        )
+
+        
+        fig.add_trace(
+            go.Scatter(
+                x=pdfAnoms.Provider,
+                y=pdfAnoms.InscClaimAmtReimbursed,
+                mode="markers",
+                marker = dict(size = 15, color = 'IndianRed', symbol = 'x'),
+                #offsetgroup="anoms",
+                #legendgroup="anoms",
+                name="Anomalies"
+            )) 
+
+        st.plotly_chart(fig, use_container_width=True)
+
+
+
+def btnSave_testFile(pdfClaims, pdfPred):
+    #--- get all providers for all anoms
+    #print("TRACE (lit_anom_superv.btnSave_testFile)  query anoms ... ", pdfPred.head(10))
+    pdfAnomProv = pdfPred[pdfPred['hasAnom?'] > 0] 
+    #pdfAnomProv = pdfAnomProv['Provider']
+
+    #--- filter claims by anomProviders
+    print("TRACE (lit_anom_superv.btnSave_testFile)  filter claims ... ")
+    pdfClaimAnom = pdfClaims[pdfClaims['Provider'].isin(pdfAnomProv['Provider'])]
+    pdfClaimNoAnom = pdfClaims[~pdfClaims['Provider'].isin(pdfAnomProv['Provider'])]
+    lngNumAnoms = len(pdfClaimAnom.index)
+    lngNumOk = len(pdfClaimNoAnom.index)
+    print("TRACE (lit_anom_superv.btnSave_testFile)  #anoms: ", lngNumAnoms, ",  !anoms: ", lngNumOk)
+
+    #--- get a sample for remaining records
+    print("TRACE (lit_anom_superv.btnSave_testFile)  sampling claims ... ")
+    pdfSave = pd.concat([pdfClaimAnom.sample(frac=0.6), pdfClaimNoAnom.sample(frac=0.1)])
+
+    print("TRACE (lit_anom_superv.btnSave_testFile)  saving ... ")
+    saveProviderTestData(pdfSave)
+
+
+def saveProviderTestData(pdfTestData):
+
+    #--- save the file
+    from datetime import date
+    import time
+    import pickle
+    strDteNow = date.today().strftime('%Y%m%d')
+    strTimeNow = time.strftime('%H%M%S')
+    strProvTestFile = libUtils.pth_data + strDteNow + strTimeNow + "_provTestSample.pkl"
+    #pd.to_pickle(pdfClaims.sample(200), strProvTestFile,  protocol=pickle.HIGHEST_PROTOCOL) 
+    pdfTestData.to_pickle(strProvTestFile, protocol=pickle.HIGHEST_PROTOCOL)
+

uix/pages/lit_anom_unsuperv.py

@@ -0,0 +1,280 @@
+#--- anomaly detection - unsupervised page
+import streamlit as st
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import plotly.graph_objects as go
+
+import lib.claims as libClaims
+import lib.providers as libProviders
+import lib.utils as libUtils
+import sys
+
+description = "Anomaly Detection - Unsupervised"
+m_kblnTraceOn = False                                  #--- enable/disable module level tracing
+
+def run():
+    #--- note:  in python, you need to specify global scope for fxns to access module-level variables 
+    global m_kblnTraceOn
+    print("\nINFO (lit_about.run)  loading ", description, " page ...") 
+
+
+    try:
+
+        #--- page settings
+        if (m_kblnTraceOn):  print("TRACE (litAnomUnSuperv.run):  Initialize Page Settings ...")
+        st.header("Claims Anomalies - Unsupervised Approach (KMeans)")
+
+
+        #--- provide file drag/drop capability
+        m_blnDisableDragDrop = False
+        if(not m_blnDisableDragDrop): 
+            #btnSave = st.button("Save")
+            pklDropped = st.file_uploader("Upload a Claims Dataset", type=["pkl"])
+            m_blnDisableDragDrop = (pklDropped is None)
+
+
+        #--- show:  raw claims data analysis
+        if (m_kblnTraceOn):  print("TRACE (litAnomUnSuperv.run):  load raw claims data ...")
+        if (m_blnDisableDragDrop):
+            pdfClaims = libClaims.load_claims(False)
+        else:
+            pdfClaims = pd.read_pickle(pklDropped)        
+
+        #--- show:  raw claims data analysis
+        if (m_kblnTraceOn):  print("TRACE (litAnomUnsuperv.run):  Show Raw Claims Dataframe ...")
+        pdfClaims = libClaims.load_claims(False)
+
+
+        #--- get unsupervised predictions
+        #pdfFeatEng = libClaims.do_featEng(pdfClaims)
+        pdfPred = libClaims.get_kmeansPredict(pdfClaims)
+        pdfSample = pdfPred.sample(100)
+        pdfSample['providerId'] = pdfSample['Provider'].str[3:].astype(np.float64)
+
+
+        #--- save this file locally as a pkl
+        #btnSave_testFile(pdfClaims, pdfPred)
+
+
+        #--- table of claims and clusters, sorted by InscClaimAmt Reimbursed
+        pdfTopClaims = pdfSample.sort_values(by=["cluster", "InscClaimAmtReimbursed"], ascending=False)
+        if (m_kblnTraceOn):  print("TRACE (litAnomUnsuperv.run):  Show $claims reimbursed by cluster ...")
+        st.markdown("(Top) Ins Claim Reimbursed by Cluster")
+        st.dataframe(pdfTopClaims)
+
+        
+        #--- chart cluster data distribution
+        chart_clusterDistr(pdfSample)
+
+
+        col1, col2, col3 = st.columns(3)
+
+
+        #--- chart KMeans clusters":  InscClaimAmtReimbursed
+        #chart_KMeansClusters(pdfSample, "Age", "InscClaimAmtReimbursed", col1)
+        #chart_KMeansClusters(pdfSample, "providerId", "InscClaimAmtReimbursed", col2)        
+        
+        chart_KMeansClusters(pdfSample, "providerId", "AdmittedDays", col1)
+        chart_KMeansClusters(pdfSample, "providerId", "DeductibleAmtPaid", col2)
+        chart_KMeansClusters(pdfSample, "providerId", "InscClaimAmtReimbursed", col3)
+
+        chart_KMeansClusters(pdfSample, "providerId", "ChronicCond_KidneyDisease", col1)
+        chart_KMeansClusters(pdfSample, "providerId", "ChronicCond_Heartfailure", col2)
+        chart_KMeansClusters(pdfSample, "providerId", "ChronicCond_ObstrPulmonary", col3)
+
+        chart_KMeansClusters(pdfSample, "AdmittedDays", "DeductibleAmtPaid", col1)
+        chart_KMeansClusters(pdfSample, "AdmittedDays", "InscClaimAmtReimbursed", col2)
+        chart_KMeansClusters(pdfSample, "DeductibleAmtPaid", "InscClaimAmtReimbursed", col3)
+
+
+
+        #--- chart cluster bars
+        #chart_KMeansBars(pdfSample, "cluster", "InscClaimAmtReimbursed", col1)
+        #chart_KMeansBars(pdfSample, "cluster", "DeductibleAmtPaid", col2)
+
+        #chart_KMeansBars(pdfSample, "cluster", "IPAnnualReimbursementAmt", col1)
+        #chart_KMeansBars(pdfSample, "cluster", "IPAnnualDeductibleAmt", col2)
+    
+        #chart_KMeansBars(pdfSample, "cluster", "OPAnnualReimbursementAmt", col1)
+        #chart_KMeansBars(pdfSample, "cluster", "OPAnnualDeductibleAmt", col2)
+    
+        #chart_KMeansBars(pdfSample, "cluster", "ChronicCond_Heartfailure", col1)
+        #chart_KMeansBars(pdfSample, "cluster", "ChronicCond_KidneyDisease", col2)
+
+    except TypeError as e:
+        print("ERROR (litAnomUnsuperv.run_typeError):  ", e)
+
+    except:
+        e = sys.exc_info()
+        print("ERROR (litAnomUnsuperv.run_genError):  ", e)        
+
+
+
+def chart_clusterDistr(pdfSample):
+    #pdfClustDistr = pdfSample['cluster'].value_counts()
+    pdfBar = pdfSample
+    pdfCluster0 = pdfBar[pdfBar['cluster'] == 0] 
+    pdfCluster1 = pdfBar[pdfBar['cluster'] == 1] 
+    pdfCluster2 = pdfBar[pdfBar['cluster'] == 2] 
+
+    kstrTitle = "(KMeans Clusters) Claims data"
+    #--- chart 
+    fig = go.Figure(
+        layout=dict(
+            legend=dict(groupclick="toggleitem"),
+            xaxis=dict(title='cluster'),
+            yaxis=dict(title='#data points')
+        )
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=pdfCluster0['cluster'],
+            y=pdfCluster0['cluster'].value_counts(),
+            name='cluster0'
+        )
+    )
+
+    if (pdfCluster1.shape[0]>0):
+        fig.add_trace(
+            go.Bar(
+                x=pdfCluster1['cluster'],
+                y=pdfCluster1['cluster'].value_counts(),
+                name='cluster1'
+            )) 
+
+    if (pdfCluster2.shape[0]>0):
+        fig.add_trace(
+            go.Bar(
+                x=pdfCluster2['cluster'],
+                y=pdfCluster2['cluster'].value_counts(),
+                name='cluster2'
+            )) 
+    st.plotly_chart(fig, use_container_width=True)
+
+
+def chart_KMeansClusters(pdfSample, strXFeature, strYFeature, stCol):
+    pdfScatter = pdfSample
+    pdfCluster0 = pdfScatter[pdfScatter['cluster'] == 0] 
+    pdfCluster1 = pdfScatter[pdfScatter['cluster'] == 1] 
+    pdfCluster2 = pdfScatter[pdfScatter['cluster'] == 2] 
+
+    kstrTitle = "(KMeans Clusters) Claims data"
+    #--- chart 
+    fig = go.Figure(
+        layout=dict(
+            legend=dict(groupclick="toggleitem"),
+            xaxis=dict(title=strXFeature),
+            yaxis=dict(title=strYFeature)
+        )
+    )
+
+    fig.add_trace(
+        go.Scatter(
+            x=pdfCluster0[strXFeature],
+            y=pdfCluster0[strYFeature],
+            text="claimId: " + pdfCluster0['ClaimID'],
+            mode='markers',
+            name='cluster0'
+        )
+    )
+
+    if (pdfCluster1.shape[0]>0):
+        fig.add_trace(
+            go.Scatter(
+                x=pdfCluster1[strXFeature],
+                y=pdfCluster1[strYFeature],
+                mode='markers',
+                name='cluster1'
+            )) 
+
+    if (pdfCluster2.shape[0]>0):
+        fig.add_trace(
+            go.Scatter(
+                x=pdfCluster2[strXFeature],
+                y=pdfCluster2[strYFeature],
+                mode='markers',
+                name='cluster2'
+            )) 
+    stCol.plotly_chart(fig, use_container_width=True)
+
+
+def chart_KMeansBars(pdfSample, strXFeature, strYFeature, stCol):
+    pdfBar = pdfSample
+    pdfCluster0 = pdfBar[pdfBar['cluster'] == 0] 
+    pdfCluster1 = pdfBar[pdfBar['cluster'] == 1] 
+    pdfCluster2 = pdfBar[pdfBar['cluster'] == 2] 
+
+    kstrTitle = "(KMeans Clusters) Claims data"
+    #--- chart 
+    fig = go.Figure(
+        layout=dict(
+            legend=dict(groupclick="toggleitem"),
+            xaxis=dict(title=strXFeature),
+            yaxis=dict(title=strYFeature)
+        )
+    )
+
+    fig.add_trace(
+        go.Bar(
+            x=pdfCluster0[strXFeature],
+            y=pdfCluster0[strYFeature],
+            name='cluster0'
+        )
+    )
+
+    if (pdfCluster1.shape[0]>0):
+        fig.add_trace(
+            go.Bar(
+                x=pdfCluster1[strXFeature],
+                y=pdfCluster1[strYFeature],
+                name='cluster1'
+            )) 
+
+    if (pdfCluster2.shape[0]>0):
+        fig.add_trace(
+            go.Bar(
+                x=pdfCluster2[strXFeature],
+                y=pdfCluster2[strYFeature],
+                name='cluster2'
+            )) 
+    stCol.plotly_chart(fig, use_container_width=True)
+
+
+
+def btnSave_testFile(pdfClaims, pdfPred):
+    #--- get all claims for all anoms
+    """     print("TRACE (lit_anom_unsuperv.btnSave_testFile)  query anoms ... ", pdfPred.head(10))
+        pdfAnomClaims = pdfPred[pdfPred['hasAnom?'] > 0] 
+        #pdfAnomProv = pdfAnomProv['Provider']
+
+        #--- filter claims by anomProviders
+        print("TRACE (lit_anom_unsuperv.btnSave_testFile)  filter claims ... ")
+        pdfClaimAnom = pdfClaims[pdfClaims['Provider'].isin(pdfAnomProv['Provider'])]
+        pdfClaimNoAnom = pdfClaims[~pdfClaims['Provider'].isin(pdfAnomProv['Provider'])]
+        lngNumAnoms = len(pdfClaimAnom.index)
+        lngNumOk = len(pdfClaimNoAnom.index)
+        print("TRACE (lit_anom_unsuperv.btnSave_testFile)  #anoms: ", lngNumAnoms, ",  !anoms: ", lngNumOk)
+
+        #--- get a sample for remaining records
+        print("TRACE (lit_anom_unsuperv.btnSave_testFile)  sampling claims ... ")
+        pdfSave = pd.concat([pdfClaimAnom.sample(frac=0.6), pdfClaimNoAnom.sample(frac=0.1)]) """
+
+    pdfSave = pdfClaims.sample(frac=0.1)
+
+    print("TRACE (lit_anom_unsuperv.btnSave_testFile)  saving ... ")
+    saveProviderTestData(pdfSave)
+
+
+def saveProviderTestData(pdfTestData):
+
+    #--- save the file
+    from datetime import date
+    import time
+    import pickle
+    strDteNow = date.today().strftime('%Y%m%d')
+    strTimeNow = time.strftime('%H%M%S')
+    strProvTestFile = libUtils.pth_data + strDteNow + strTimeNow + "_claimsTestSample.pkl"
+    #pd.to_pickle(pdfClaims.sample(200), strProvTestFile,  protocol=pickle.HIGHEST_PROTOCOL) 
+    pdfTestData.to_pickle(strProvTestFile, protocol=pickle.HIGHEST_PROTOCOL)

uix/pages/lit_claimAnalysis.py

@@ -0,0 +1,75 @@
+#--- claim analysis page
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+
+import lib.claims as libClaims
+
+description = "Claim Analysis"
+m_kbln_traceOn = False                                  #--- enable/disable module level tracing
+
+
+def run():
+    #--- note:  in python, you need to specify global scope for fxns to access module-level variables 
+    global m_kbln_traceOn
+
+    try:
+
+        #--- page settings
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  Initialize Page Settings ...")
+        st.header("Claims Analysis")
+
+
+        #--- show:  raw claims data analysis
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  Show Raw Claims Dataframe ...")
+        dfClaims = libClaims.load_claims(False) 
+        #dfClaims = libClaims.loadPkl_testClaims()       #--- note:  a large dataset;  reduce before render
+        dfRaw = dfClaims.sample(25)
+        st.markdown("(Sample) Raw Claims Data:  Providers, Beneficiaries, Physicians, Procedures, etc")
+        st.dataframe(dfRaw)
+
+
+        #--- show:  data grouped by provider
+        pdfClaimsByProvider = dfClaims.groupby(
+            by=["Provider"], as_index=False).agg(
+                {"ClaimID":"count", "InscClaimAmtReimbursed":"sum", "DeductibleAmtPaid":"sum"}
+            )    
+        st.markdown("(Sample) Raw Claims Data:  Grouped by Provider")
+        st.dataframe(pdfClaimsByProvider.sample(25))
+
+        #--- show:  bar charts
+        col1, col2 = st.columns(2)
+
+        #--- show $claims reimbursed by provider
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  Show $claims reimbursed by provider ...")
+        pdfTopClaimsByProv = dfClaims.nlargest(10, "InscClaimAmtReimbursed")
+        fig = px.bar(pdfTopClaimsByProv,
+            x="Provider", y="InscClaimAmtReimbursed", title="$ Claims by Provider")
+        #col1.markdown("(Sample) $Claims Reimbursed by Provider")
+        col1.plotly_chart(fig, use_container_width=True)
+
+        #--- #claims reimbursed by provider
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  Show #claims reimbursed by provider ...")
+        #pdfMaxClaimsByProv = dfClaims.groupby(['Provider'])['ClaimID'].count()
+        pdfClaimCountByProv = dfClaims.groupby(
+            by=["Provider"], as_index=False).agg(
+                {"ClaimID": "count"}
+            )    
+        pdfClaimCountByProv = pdfClaimCountByProv.nlargest(10, "ClaimID")
+        fig = px.bar(pdfClaimCountByProv,
+            x="Provider", y="ClaimID", title="# Claims by Provider", barmode="group")
+        #col2.markdown("(Sample) #Claims Reimbursed by Provider")           #--- just to even out the display
+        col2.plotly_chart(fig, use_container_width=True)
+
+
+        #--- TODO:  (optimization) create a single group by dataframe;  try not to recreate for each chart 
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  Show top $deductible paid by provider ...")
+        pdfDedAmtPaid = dfClaims.nlargest(10, "DeductibleAmtPaid")
+        fig = px.bar(pdfDedAmtPaid,
+            x="Provider", y="DeductibleAmtPaid", title="Deductible Paid by Provider")
+        col1.plotly_chart(fig, use_container_width=True)
+    
+        if (m_kbln_traceOn):  print("TRACE (litClaimAnalysis.run):  end of fxn ...")
+
+    except TypeError as e:
+        print("ERROR (litClaimAnalysis.run):  ", e)

uix/pages/lit_home.py

@@ -0,0 +1,41 @@
+#--- about page
+import streamlit as st
+
+description = "Home"
+def run():
+
+    print("\nINFO (lit_home.run)  loading ", description, " page ...") 
+
+
+    st.markdown('### Home')
+    st.markdown('### MLE10 Capstone:  Healthcare Anomaly Detection')
+    st.markdown('\
+        Healthcare fraud is an expensive white-collar crime in the US and leads to an \
+        increase in healthcare premiums, and a reduction in quality and access to care.\
+        The National Health Care Anti-Fraud Association conservatively estimates that \
+        about 3 percent of US healthcare spending is lost to fraud per year ($300 billion \
+        approximately).')
+    
+    st.markdown('\
+        Machine Learning techniques can identify current and evolving anomalies in claims \
+        data.  As fraud becomes more sophisticated across an increasing number of annual \
+        transactions, an ML solution provides an opportunity to greatly reduce the effort, \
+        time and associated cost spent in identifying claims anomalies, and recouping any \
+        misappropriated funds. ')
+    
+    st.markdown('\
+        To illustrate the capabilities of Machine Learning to identify claims anomalies, \
+        this capstone project team has developed two solutions:  \
+        \n\t - a supervised Logistic Regression Model to identify potential anomalies at \
+             the provider level \
+        \n\t - an unsupervised KMeans Clustering Model to identify potential anomalies \
+                at the claim level.')
+
+    st.markdown(
+        """
+
+            Home page
+        
+        """,
+            unsafe_allow_html=True,
+        )

uix/pages/lit_modelPerf.py

@@ -0,0 +1,6 @@
+description = "Model Performance"
+
+def run():
+    import streamlit as st
+    import pandas as pd
+    import plotly.express as px