ready for deploy to hugspace v0.1.5

.dockerignore

@@ -1,21 +0,0 @@
-
-
-#--- ignore select binary files/folders
-bin/images/sample*
-#---bin/models/*.pth
-bin/models/*.zip
-bin/models/deeplabv3*vhflip30/*.pth
-bin/testing
-_ignore
-.vscode
-cicd_workflows
-
-#--- ignore all local data files; preserve/recreate folder structure
-data/demo_tiles/*.tiff
-data/tiles
-data/wsi
-
-#--- ignore all doc files
-docs
-notebooks
-preso

.gitattributes

@@ -1,46 +0,0 @@
-#--- Note:  omdena github does not allow git lfs.  Disable.
-#--- Note:  huggingface does not allow _any_ binaries;  git lfs has to be used
-#bin/models/*.pth filter=lfs diff=lfs merge=lfs -text
-#bin/models/deeplab*vhflip30/model_a* filter=lfs diff=lfs merge=lfs -text
-#bin/images/* filter=lfs diff=lfs merge=lfs -text
-#data/demo_tiles/raw/* filter=lfs diff=lfs merge=lfs -text
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-*.tiff filter=lfs diff=lfs merge=lfs -text
-data/demo_tiles/raw/*.tiff filter=lfs diff=lfs merge=lfs -text
-bin/models/deeplabv3*vhflip30/model_a* filter=lfs diff=lfs merge=lfs -text
-*model_a* filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-bin/models/model.pth filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -21,4 +21,8 @@ EXPOSE 39500
 #   docker run -it -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaHcc_demo kidcoconut73/img_stm_omdenasaudi_hcc:demo   
 
 #--- start an existing container
-#   docker start -it ctr_stmOmdenaHcc_demo
+#   docker start -it ctr_stmOmdenaHcc_demo
+
+#--- deploy to huggingspace
+#   overwrite Dockerfile;  cp Dockerfile.hug Dockerfile
+#   push to Huggingspace;  

Dockerfile.hug

@@ -1,28 +0,0 @@
-#--- build the image from dockerHub
-FROM kidcoconut73/img_stm_omdenasaudi_hcc:0.1.4
-#FROM kidcoconut73/img_stm_omdenasaudi_hcc:demo
-
-
-#--- for streamlit;  external 49400;  internal 39400
-#--- for fastapi;  external 49500;  internal 39500
-#--- for huggingface;  assume 1:1 mapping between internal and external ports;  and only one port can truly be exposed
-EXPOSE 39400
-EXPOSE 39500
-
-#--- build/rebuild the image from an alt Dockerfile
-#	docker build -t foo/bar -f /path/to/Dockerfile /path/to/
-#	docker build -t img_stm_omdenasaudi_hcc:0.1.4 -f Dockerfile.hub ./
-
-#--- create a container from an image (without running it)
-#   docker create -it -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaHcc_demo kidcoconut73/img_stm_omdenasaudi_hcc:demo
-
-#--- to run the container (interactive) from the image;  specific port mapping (-p) vs any available port mapping (-P)
-#   docker run -it -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaHcc_demo img_stm_omdenasaudi_hcc:0.1.4                    
-#   docker run -it -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaHcc_demo kidcoconut73/img_stm_omdenasaudi_hcc:demo   
-
-#--- start an existing container
-#   docker start -it ctr_stmOmdenaHcc_demo
-
-#--- deploy to huggingspace
-#   overwrite Dockerfile;  cp Dockerfile.hug Dockerfile
-#   push to Huggingspace;  

Dockerfile.local

@@ -1,86 +0,0 @@
-#--- PREREQS:  
-#   - create a local folder dedicated to WSI image mgmt:  (docker pwd)/data 
-#   - populate the folder with raw data, wsi and tiles
-#   - docker run --name <name> -v <local folder>
-
-#--- utilize a light linux distro for python apps
-FROM python:3.10.9-slim-bullseye
-
-RUN pip install --no-cache-dir --upgrade pip
-RUN apt-get update -y && apt-get install -y ffmpeg libsm6 libxext6
-
-# Set up a new user named "user" with user ID 1000
-# Switch to the "user" user
-RUN useradd -m -u 1000 user
-USER user
-
-# Set home to the user's home directory
-ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH
-
-#--- set docker image working directory to /app
-WORKDIR $HOME/app
-
-# Try and run pip command after setting the user with `USER user` to avoid permission issues with Python
-#RUN pip install --no-cache-dir --upgrade pip
-
-
-#--- copy only the requirements.txt file
-#--- set docker image working directory to /app
-#--- Not:  this is reorg'd in an attempt to reduce the rebuilding of layers
-COPY --chown=user ./requirements.txt $HOME/app/requirements.txt
-COPY --chown=user ./packages.txt $HOME/app/packages.txt
-
-#--- install all lib dependencies into the image
-#RUN pip3 install -r $HOME/app/requirements.txt
-RUN pip install --no-cache-dir --upgrade -r $HOME/app/requirements.txt
-#RUN apt-get update -y && apt-get install -y ffmpeg libsm6 libxext6
-
-#--- copy all files from the local pwd to the docker image /app folder
-#--- .dockerignore:  ensure no local data folders or files (images) are copied into the docker image/container 
-COPY --chown=user . $HOME/app
-
-#--- for streamlit;  external 49400;  internal 39400
-#       localExec:  (from root folder) streamlit run app.py --server.port=39400 --server.maxUploadSize=2000
-EXPOSE 49400         
-#CMD ["streamlit", "run", "app.py", "--server.port=39400", "--server.maxUploadSize=2000"]
-
-#--- for fastapi;  external 49500;  internal 39500
-#       localExec:  (from root folder) uvicorn main:app --reload --workers 1 --host 0.0.0.0 --port 39500 
-EXPOSE 49500        
-#CMD ["uvicorn", "main:app", "--reload", "--host=0.0.0.0", "--port=39500"]
-
-#--- start streamlit and fastapi from a helper utility script
-#CMD ./util_startLocal_streamlitFastApi.sh
-CMD $HOME/app/scripts/docker/util_docker_preRun.sh
-
-
-#--- to build/rebuild the image;  make sure you stop and remove the container if you are replacing/upgrading;  or change the version tag# from 0.1
-#    docker build -t img_stm_omdenasaudi_hcc:0.1 .
-#    docker build -t img_stm_omdenasaudi_hcc:0.1.3 .
-
-#--- build/rebuild the image from an alt Dockerfile
-#	docker build -t foo/bar -f /path/to/Dockerfile /path/to/
-#	docker build -t img_stm_omdenasaudi_hcc:0.1.4 -f Dockerfile.hub ./
-
-#--- to tag the image prior to push to DockerHub;  docker login and then register user/image:tag
-#--- to push this image to DockerHub, example based on the repo:  kidcoconut73/img_stm_omdenasaudi_hcc
-#    docker tag img_stm_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:demo
-#    docker tag img_stm_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:0.1
-#    docker push kidcoconut73/img_stm_omdenasaudi_hcc:demo
-
-#--- to run the container from the image;  specific port mapping (-p) vs any available port mapping (-P)
-#    docker run -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaSaudiHcc -v ./data:/app/data img_stm_omdenasaudi_hcc:0.1
-#    docker run -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaSaudiHcc img_stm_omdenasaudi_hcc:0.1
-#    docker run -P --name ctr_stmOmdenaHcc img_stm_omdenasaudi_hcc:0.1                   #--- open all ports defined by Docker EXPOSE
-
-#--- to run the container in interactive mode (local debug)
-#    docker run -it -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaSaudiHcc_013 img_stm_omdenasaudi_hcc:0.1.3
-
-#--- ISSUE:  uvicorn bug does not allow ctl-C break of fastapi through terminal
-#--- WORKAROUND:  you have to run a docker or docker compose kill cmd;  eg docker kill <ctr_name>
-
-
-#---    Docker build log
-#       from python:3.10.9-slim-bullseye                    size:  4.21gb       time:  >yyys
-#       from python:3.10.9-slim-bullseye                    size:  4.21gb       time:  >3500s	:0.13	expose streamlit 49400

app.py

@@ -1,23 +0,0 @@
-'''
-    toExecute:  (from root app folder) ... streamlit run app.py
-'''
-import streamlit as st
-import uix.lit_sidebar as litSideBar
-
-
-#--- streamlit:  specify title and logo
-st.set_page_config(
-            page_title='Omdena Saudi Arabia - Liver HCC Diagnosis with XAI', 
-            #page_icon='https://cdn.freebiesupply.com/logos/thumbs/1x/nvidia-logo.png', 
-            layout="wide")
-st.header('\
-    Detecting Liver Cancer from Histopathology WSI \
-    using Deep Learning and Explainability (XAI)\
-')
-st.markdown('#### Dr. Shaista Hussain (Saudi Arabia Chapter Lead)')
-st.markdown("##### Iain McKone (Deployment Lead) [LinkedIn](%s)" % "https://linkedin.com/in/iainmckone")
-st.markdown('---')
-
-
-#--- streamlit:  add a sidebar 
-litSideBar.init()

bin/models/deeplabv3_r50_full_train_dataset_80-20_split_10epochs_no-norm_vhflip30/version

@@ -1 +0,0 @@
-3

bin/models/model.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9029cfef04ad8f66b1b59f7417ec0f2eb6721198fb8a31f32dea75d87e1168a8
-size 168385213

config.toml

@@ -1,3 +0,0 @@
-[server]
-
-maxUploadSize = 2000		#--- increased from default 200MB to 2000MB

lib/fctTile.py

@@ -1 +0,0 @@
-#--- factory class for tile operations

lib/fctWsi.py

@@ -1 +0,0 @@
-#--- factory class for wsi operations

lib/models/mdl_autoenc.py

@@ -1,55 +0,0 @@
-import pandas as pd
-import numpy as np
-from sklearn.decomposition import PCA 
-import lib.utils as libPaths
-import pickle
-
-
-m_kstrFile = __file__
-m_kstrDataPath = libPaths.pth_data
-m_kstrBinModelPath = libPaths.pth_binModels
-m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_unsuperv_colab.pkl'
-m_kstrEncModelPath = m_kstrBinModelPath + 'enc_keras_seq/'
-
-
-#--- Supervised:  autoencoder - Principal Component Analysis
-def load_encFromKeras():
-    from tensorflow import keras
-    mdlAnoms = keras.models.load_model(m_kstrEncModelPath)
-    return mdlAnoms
-
-
-def load_pcaFromPkl():
-    with open(m_kstrPcaModelPath, 'rb') as filPkl:
-        # load using pickle de-serializer
-        mdlAnoms = pickle.load(filPkl)
-    return mdlAnoms
-
-
-def save_encToKeras(mdlAnoms):
-    mdlAnoms.save(m_kstrEncModelPath)
-
-
-
-def predict(pdfScaled):
-    
-    #--- Pre:  Transforming train and test dataframes based on PCA
-    mdlPCA = load_pcaFromPkl()         #--- this is a pre-fit model based on training
-    npaPca = mdlPCA.transform(pdfScaled)
-    print("INFO (" + m_kstrFile + ".predict)  npaPca.shape:  ", npaPca.shape)
-
-
-    #--- predict on unseen data
-    mdlEnc = load_encFromKeras()
-    npaPredict = mdlEnc.predict(npaPca[:,:29])
-    print("INFO (" + m_kstrFile + ".predict)  npaPredict.shape:  ", npaPredict.shape)
-    #--- expected:  297, 29?
-    return npaPredict
-    
-
-""" 
-def train(pdfTrainData):
-    mdlAnoms = PCA()                        #---- TODO:  this is Keras Sequential
-    mdlAnoms.fit(pdfTrainData.values)
-    save_encToKeras(mdlAnoms)
-    return mdlAnoms """

lib/models/mdl_kmeans.py

@@ -1,155 +0,0 @@
-from sklearn.cluster import KMeans
-import lib.utils as libPaths
-import pickle
-import pandas as pd
-
-
-m_kstrFile = __file__
-m_kstrDataPath = libPaths.pth_data
-m_kstrBinModelPath = libPaths.pth_binModels
-
-#m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_kmeans_unsuperv_colab.pkl'
-#m_kstrPcaModelPath = m_kstrBinModelPath + 'pca_kmeans_unsuperv_colab_v1.2.1.pkl'
-m_kstrPcaModelPath_111 = m_kstrBinModelPath + 'claims_pca_v1.1.1_27cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
-m_kstrPcaModelPath_121 = m_kstrBinModelPath + 'claims_pca_v1.2.1_27cols.pkl'
-m_kstrPcaModelPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_pca_py3816_sk111hp_27cols.pkl'
-m_kstrPcaModelPath = m_kstrPcaModelPath_claims_py3816_sk111hp
-
-#m_kstrKmeansModelPath = m_kstrBinModelPath + 'kmeans_unsuperv_colab.pkl'
-#m_kstrKmeansModelPath = m_kstrBinModelPath + 'kmn_unsuperv_colab_v1.2.1.pkl'
-m_kstrModelPath_111 = m_kstrBinModelPath + 'claims_kmn_v1.1.1_22cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
-m_kstrModelPath_121 = m_kstrBinModelPath + 'claims_kmn_v1.2.1_22cols.pkl'
-m_kstrModelPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_kmn_py3816_sk111hp_22cols.pkl'
-m_kstrKmeansModelPath = m_kstrModelPath_claims_py3816_sk111hp
-
-m_blnTraceOn = True
-
-
-#--- unsupervised:  Logistic Regession
-def load_pcaFromPkl():
-    with open(m_kstrPcaModelPath, 'rb') as filPkl:
-        mdlAnoms = pickle.load(filPkl)
-    return mdlAnoms
-
-
-#--- unsupervised:  KMeans
-def load_kmeansFromPkl():
-    with open(m_kstrKmeansModelPath, 'rb') as filPkl:
-        mdlAnoms = pickle.load(filPkl)
-    return mdlAnoms
-
-
-def save_pcaToPkl(mdlAnoms):
-    with open(m_kstrPcaModelPath, 'wb') as filPkl:
-        pickle.dump(mdlAnoms, filPkl)
-    return mdlAnoms
-
-
-def save_kmeansToPkl(mdlAnoms):
-    with open(m_kstrKmeansModelPath, 'wb') as filPkl:
-        pickle.dump(mdlAnoms, filPkl)
-    return mdlAnoms
-
-
-
-#--- determine which points can be labelled against which clusters
-def predict(pdfScaled):
-    #--- load a persisted fit kmeans model
-    #--- predict will assign labels onto a similarly scaled data frame 
-
-
-    #--- Note:  reverse chron through the code ...
-    #---        4. KMeans was fit on X-reduced (22 cols)
-    #---        3. X_reduced was a reduced column set of X-scaled (27 -> 22;  Dropped 5 cols:  DeadOrNot; and hotEncoded Gender and Race)
-    #---        2. x_scaled was transformed through stdScaler
-    #---        1. StdScaler was fit on X to produce X-scaled (X has 27 cols)
-    pdfReduced = pdfScaled[['InscClaimAmtReimbursed', 'DeductibleAmtPaid',
-        'AdmittedDays', 'RenalDiseaseIndicator', 'NoOfMonths_PartACov',
-        'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
-        'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
-        'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
-        'ChronicCond_Depression', 'ChronicCond_Diabetes',
-        'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
-        'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
-        'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
-        'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age']]
-
-    #--- prefit Kmeans clustering - was fit on trained pdfReduced
-    #--- Note:  if we want to understand how kmeans performs on test/prod data, we need to predict
-    mdlKMeans = load_kmeansFromPkl()
-    #ndaPredict = mdlKMeans.predict(pdfScaled)         #20230208:  ValueError: X has 27 features, but KMeans is expecting 22 features as input.
-    ndaPredict = mdlKMeans.predict(pdfReduced)       #ValueError: X has 22 features, but KMeans is expecting 27 features as input.
-    return ndaPredict
-
-
-#--- feat eng
-def do_featEng(pdfLoaded, blnIsTrain=False, hasGroupByProviderCols=True):
-    print("INFO (mdl_kmeans.doFeatEng):  blnIsTrain, ", blnIsTrain)
-
-    #--- columns_to_remove
-    aryColsToDrop = ['BeneID', 'ClaimID', 'ClaimStartDt','ClaimEndDt','AttendingPhysician',
-                     'OperatingPhysician', 'OtherPhysician', 'ClmDiagnosisCode_1',
-                     'ClmDiagnosisCode_2', 'ClmDiagnosisCode_3', 'ClmDiagnosisCode_4',
-                     'ClmDiagnosisCode_5', 'ClmDiagnosisCode_6', 'ClmDiagnosisCode_7',
-                     'ClmDiagnosisCode_8', 'ClmDiagnosisCode_9', 'ClmDiagnosisCode_10',
-                     'ClmProcedureCode_1', 'ClmProcedureCode_2', 'ClmProcedureCode_3',
-                     'ClmProcedureCode_4', 'ClmProcedureCode_5', 'ClmProcedureCode_6',
-                     'ClmAdmitDiagnosisCode', 'AdmissionDt',
-                     'DischargeDt', 'DiagnosisGroupCode','DOB', 'DOD',
-                     'State', 'County']
-    pdfFeatEng = pdfLoaded.drop(columns=aryColsToDrop, axis=1)
-
-    #--- flag categorical cols
-    pdfFeatEng.Gender = pdfFeatEng.Gender.astype('category')
-    pdfFeatEng.Race = pdfFeatEng.Race.astype('category')
-
-    #--- one-hot-encoding
-    pdfFeatEng = pd.get_dummies(pdfFeatEng, columns=['Gender', 'Race'], drop_first=True)
-    if (blnIsTrain):
-        #--- one-hot encode the potential fraud column (for training data only)
-        try:
-            #print("INFO (claims.doFeatEng):  one-hot encoding potential fraud")
-            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'Yes', 'PotentialFraud'] = 1
-            pdfFeatEng.loc[pdfFeatEng['PotentialFraud'] == 'No', 'PotentialFraud'] = 0
-        except KeyError:
-            #--- likely column not found; invalid fxn call
-            print("ERROR (claims.doFeatEng):  Potential Fraud col not found")
-
-    pdfFeatEng.loc[pdfFeatEng['RenalDiseaseIndicator'] == 'Y', 'RenalDiseaseIndicator'] = 1 
-    pdfFeatEng['DeductibleAmtPaid'].fillna(0, inplace=True)
-    pdfFeatEng['AdmittedDays'].fillna(0, inplace=True)
-
-    #--- check for correlated cols
-
-    #--- add new features to assist with predictions
-    if (hasGroupByProviderCols):
-        pdfFeatEng['InscClaimReimbursement_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['InscClaimAmtReimbursed'].transform('mean') 
-        pdfFeatEng['DeductibleAmtPaid_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['DeductibleAmtPaid'].transform('mean')
-        
-        pdfFeatEng['IPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualReimbursementAmt'].transform('mean')
-        pdfFeatEng['IPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['IPAnnualDeductibleAmt'].transform('mean')
-
-        pdfFeatEng['OPAnnualReimbursementAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualReimbursementAmt'].transform('mean')
-        pdfFeatEng['OPAnnualDeductibleAmt_ProviderAvg'] = pdfFeatEng.groupby(['Provider'])['OPAnnualDeductibleAmt'].transform('mean')
-    return pdfFeatEng
-
-
-def fit(pdfScaled):
-    #--- determine the centroids of the kmeans clusters
-    #--- refit kmeans clustering according to the pre-scaled data provided
-    #--- note:  this all assumes that the nature of the data and the number of clusters remain unchanged
-    m_klngNumClusters = 3
-    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit)  instantiate KMeans ...")
-    mdlKMeans = KMeans(n_clusters=m_klngNumClusters, max_iter=50, random_state=2022)            #--- #clusters was learned from training    
-    
-    if (m_blnTraceOn): print("TRACE (" + m_kstrFile + ".fit)  fitting data (scaled) ...")
-    mdlKMeans.fit(pdfScaled)           #--- fit on test/prod data
-    
-    return mdlKMeans                    #--- this ibject will give us all results based on kmeans
-
-
-def train(pdfTrainData):
-    mdlAnoms = KMeans(n_clusters=3, max_iter=50, random_state=2022)
-    mdlAnoms.fit(pdfTrainData.values)
-    save_kmeansToPkl(mdlAnoms)
-    return mdlAnoms

lib/models/mdl_logR.py

@@ -1,41 +0,0 @@
-from sklearn.linear_model import LogisticRegressionCV
-import lib.utils as libPaths
-import pickle
-
-
-m_kstrFile = __file__
-m_kstrDataPath = libPaths.pth_data
-m_kstrBinModelPath = libPaths.pth_binModels
-m_kstrModelPath = m_kstrBinModelPath + 'lgr_model_colab.pkl'
-
-
-#--- Supervised:  Logistic Regession
-def load_fromPkl():
-    with open(m_kstrModelPath, 'rb') as filPkl:
-        mdlAnoms = pickle.load(filPkl)
-    return mdlAnoms
-
-
-
-def save_toPkl(mdlAnoms):
-    with open(m_kstrModelPath, 'wb') as filPkl:
-        pickle.dump(mdlAnoms, filPkl)
-    return mdlAnoms
-
-
-
-def predict(npaData):
-    #--- input:  numpy.ndarray of feature eng, and scaled data 
-    mdlAnoms = load_fromPkl()
-    npaPredict = mdlAnoms.predict(npaData)
-
-    print("INFO (npaPredict.shape):  ", npaPredict.shape)
-    return npaPredict
-
-
-
-def train(pdfTrainData):
-    mdlAnoms = LogisticRegressionCV()
-    mdlAnoms.fit(pdfTrainData.values)
-    save_toPkl(mdlAnoms)
-    return mdlAnoms

lib/models/mdl_svm.py

@@ -1,40 +0,0 @@
-from sklearn.svm import LinearSVC
-import lib.utils as libPaths
-import pickle
-
-
-m_kstrFile = __file__
-m_kstrDataPath = libPaths.pth_data
-m_kstrBinModelPath = libPaths.pth_binModels
-m_kstrModelPath = m_kstrBinModelPath + 'svm_model_colab.pkl'
-
-
-#--- Supervised:  Support Vector Machines
-def load_fromPkl():
-    with open(m_kstrModelPath, 'rb') as filPkl:
-        mdlAnoms = pickle.load(filPkl)
-    return mdlAnoms
-
-
-
-def save_toPkl(mdlAnoms):
-    with open(m_kstrModelPath, 'wb') as filPkl:
-        pickle.dump(mdlAnoms, filPkl)
-    return mdlAnoms
-
-    
-
-def predict(npaData):
-    #--- input:  numpy.ndarray of feature eng, and scaled data 
-    mdlAnoms = load_fromPkl()
-    npaPredict = mdlAnoms.predict(npaData)
-    print("INFO (" + m_kstrFile + ".predict)  npaPredict.shape:  ", npaPredict.shape)
-    return npaPredict
-    
-
-
-def train(pdfTrainData):
-    mdlAnoms = LinearSVC()
-    mdlAnoms.fit(pdfTrainData.values)
-    save_toPkl(mdlAnoms)
-    return mdlAnoms

lib/models/mdl_utils.py

@@ -1,256 +0,0 @@
-import pandas as pd
-import pickle
-import lib.utils as libPaths
-
-m_blnTraceOn = False
-
-#--- load, merge data from file
-m_kstrDataPath = libPaths.pth_data
-m_kstrModelPath = libPaths.pth_model
-m_kstrBinModelPath = libPaths.pth_binModels
-
-#m_kstrScalerPath_claims = m_kstrBinModelPath + 'stdClaims_scaler_colab.pkl'         #--- does not work for scaling claims data;  from v1.0.2; using 1.1.1
-#m_kstrScalerPath_claims2 = m_kstrBinModelPath + 'std_scaler_unsuperv_colab.pkl'     #--- does not work; expects 32 features 
-#m_kstrScalerPath_claims = m_kstrBinModelPath + 'stdClaims_scaler_colab_v1.2.1.pkl'   
-m_kstrScalerPath_claims111 = m_kstrBinModelPath + 'claims_stdScaler_v1.1.1_27cols.pkl'
-m_kstrScalerPath_claims121 = m_kstrBinModelPath + 'claims_stdScaler_v1.2.1_27cols.pkl'
-m_kstrScalerPath_claims_py3816_sk111hp = m_kstrBinModelPath + 'claims_stdScl_py3816_sk111hp_27cols.pkl'
-m_kstrScalerPath_claims = m_kstrScalerPath_claims_py3816_sk111hp
-
-m_kstrScalerPath_providers111 = m_kstrBinModelPath + 'prov_stdScaler_v1.1.1_32cols.pkl'
-m_kstrScalerPath_providers121 = m_kstrBinModelPath + 'prov_stdScaler_v1.2.1_32cols.pkl'
-m_kstrScalerPath_prov_py3816_sk111 = m_kstrBinModelPath + 'prov_stdScl_py3816_sk111_32cols.pkl'
-m_kstrScalerPath_prov_py3816_sk111hp = m_kstrBinModelPath + 'prov_stdScl_py3816_sk111hp_32cols.pkl'
-m_kstrScalerPath_prov = m_kstrScalerPath_prov_py3816_sk111hp
-
-m_kstrScalerPath_providers_superv = m_kstrBinModelPath + 'gbc_scaler.pkl'
-m_kstrScalerPath_providers_train = m_kstrBinModelPath + "stdProvider_scaler.pkl" 
-   
-
-
-def doProviders_stdScaler(pdfFeatEng, blnIsTrain=False, hasGroupByProviderCols=True):
-    print("INFO (claims.do_stdScaler):  blnIsTrain, ", blnIsTrain)
-
-    #--- Note:  prediction runs on X_val
-    '''
-    #--- WARN:  The default value of numeric_only in DataFrameGroupBy.sum is deprecated. 
-    #           In a future version, numeric_only will default to False. Either specify 
-    #           numeric_only or select only columns which should be valid for the function.
-    '''
-
-    #--- WARN:  this code groups all data by provider;  any predictions will also be by provider
-    pdfGroupBy = pdfFeatEng
-    if (hasGroupByProviderCols):
-        pdfGroupBy = pdfFeatEng.groupby(['Provider'], as_index=False).agg('sum')
-    
-    X = pdfGroupBy
-
-    try:
-        X = X.drop(columns=['Provider'], axis=1)        #--- cannot scale;  text
-    except KeyError:
-        #--- likely column not found; invalid fxn call
-        print("ERROR (mdlUtils.doProviders_stdScaler):  Provider col not found")
-
-    try:
-        X = X.drop(columns=['PotentialFraud'], axis=1)  
-    except KeyError:
-        #--- likely column not found; invalid fxn call
-        if (blnIsTrain):  print("ERROR (mdlUtils.doProviders_stdScaler):  Potential Fraud col not found")
-
-
-    #--- apply std scaler
-    #--- WARN:  scaling is also grouped by provider
-    if (m_blnTraceOn):  print("INFO (mdlUtils.doProviders_stdScaler)  cols: ", X.columns)           #--- 32cols
-    X_std = fitProviders_txfStdScaler(X, blnIsTrain)
-    return X_std
-
-
-
-def doClaims_stdScaler(pdfFeatEng, blnIsTrain=False):
-    print("INFO (mdlUtils.doClaims_stdScaler):  blnIsTrain, ", blnIsTrain)
-
-    #--- Note:  prediction runs on X_val
-    '''
-    #--- WARN:  The default value of numeric_only in DataFrameGroupBy.sum is deprecated. 
-    #           In a future version, numeric_only will default to False. Either specify 
-    #           numeric_only or select only columns which should be valid for the function.
-    '''
-
-    #--- WARN:  this code groups all data by provider;  any predictions will also be by provider
-    X = pdfFeatEng
-
-    try:
-        X = X.drop(columns=['Provider'], axis=1)        #--- cannot scale;  text
-    except KeyError:
-        #--- likely column not found; invalid fxn call
-        print("ERROR (mdlUtils.do_stdScaler):  Provider col not found")
-
-    try:
-        X = X.drop(columns=['PotentialFraud'], axis=1)  
-    except KeyError:
-        #--- likely column not found; invalid fxn call
-        if (blnIsTrain):  print("ERROR (mdlUtils.do_stdScaler):  Potential Fraud col not found")
-
-
-    #--- apply std scaler
-    #--- WARN:  scaling is also grouped by provider
-    #print("INFO (mdlUtils.doClaims_stdScaler)  cols: ", X.columns)
-    X_std = fitClaims_txfStdScaler(X, blnIsTrain)
-    return X_std
-
-
-
-def doProviders_stdScaler_toPdf(npaScaled):
-    #--- NOTE:  the list of cols came from doProvider_stdScaler; print(X.columns)
-    aryCols = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'AdmittedDays',
-       'NoOfMonths_PartACov', 'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
-       'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
-       'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
-       'ChronicCond_Depression', 'ChronicCond_Diabetes',
-       'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
-       'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
-       'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
-       'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age', 'DeadOrNot',
-       'Gender_2', 'Race_2', 'Race_3', 'Race_5',
-       'ClaimReimbursement_ProviderAvg',
-       'ClaimReimbursement_AttendingPhysician',
-       'ClaimReimbursement_OperatingPhysician',
-       'DeductibleAmtPaid_ProviderAvg', 'DeductibleAmtPaid_AttendingPhysician',
-       'DeductibleAmtPaid_OperatingPhysician']
-
-    #npaScaled = do_stdScaler(pdfFeatEng)
-    pdfScaled = pd.DataFrame(npaScaled, columns=aryCols)
-    return pdfScaled
-
-
-
-def doClaims_stdScaler_toPdf(npaScaled):
-    #--- NOTE:  the list of cols came from doClaims_stdScaler; print(X.columns)
-    aryCols = ['InscClaimAmtReimbursed', 'DeductibleAmtPaid', 'AdmittedDays',
-       'RenalDiseaseIndicator', 'NoOfMonths_PartACov', 'NoOfMonths_PartBCov', 'ChronicCond_Alzheimer',
-       'ChronicCond_Heartfailure', 'ChronicCond_KidneyDisease',
-       'ChronicCond_Cancer', 'ChronicCond_ObstrPulmonary',
-       'ChronicCond_Depression', 'ChronicCond_Diabetes',
-       'ChronicCond_IschemicHeart', 'ChronicCond_Osteoporasis',
-       'ChronicCond_rheumatoidarthritis', 'ChronicCond_stroke',
-       'IPAnnualReimbursementAmt', 'IPAnnualDeductibleAmt',
-       'OPAnnualReimbursementAmt', 'OPAnnualDeductibleAmt', 'Age', 'DeadOrNot',
-       'Gender_2', 'Race_2', 'Race_3', 'Race_5']
-
-    #npaScaled = do_stdScaler(pdfFeatEng)
-    pdfScaled = pd.DataFrame(npaScaled, columns=aryCols)
-    return pdfScaled
-
-
-
-
-def fitClaims_stdScaler(pdfData, blnIsTrain=False):
-    #--- apply scaler
-    #--- WARN:  scaling is not grouped by provider
-    from sklearn.preprocessing import StandardScaler
-
-    #--- note:  this is a numpy.ndarray
-    #--- we need to fit the scaler, and then save as a pkl file
-    #strScalerPath = m_kstrScalerPath_claims
-    strScalerPath = m_kstrScalerPath_claims
-#    strScalerPath = m_kstrBinModelPath + "stdClaims_scaler_colab.pkl"
-    if (m_blnTraceOn):  print("INFO (lib.model.fitClaims_stdScalar):  ", strScalerPath)
-    if (blnIsTrain):
-        scaler = StandardScaler()
-        sclFit = scaler.fit(pdfData)
-        #--- if we train locally;  write out to gbc_scalar.pkl
-        #--- we do not want to overwrite the colab version used for test
-        strScalerPath = m_kstrBinModelPath + "stdClaims_scaler.pkl"
-        if (m_blnTraceOn):  print("INFO (lib.model.fit_stdScalar)  Using local pkl for Train: ", strScalerPath)
-        with open(strScalerPath, 'wb') as filPkl:
-            pickle.dump(sclFit, filPkl)
-    else:
-        #--- we need to load the pkl file
-        import sklearn
-        if (m_blnTraceOn):  print("INFO (lib.model.fit_stdScalar)  Using colab pkl for Test: ", strScalerPath)
-        with open(strScalerPath, 'rb') as filPkl:
-            sclFit = pickle.load(filPkl)
-        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sclFit.type: ", type(sclFit))
-
-        #--- testing
-        scaler = StandardScaler()
-        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  StdScaler.version: ", scaler.__getstate__()['_sklearn_version'])
-        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sclFit.version: " , sclFit.__getstate__()['_sklearn_version'])
-        if (m_blnTraceOn):  print("TRACE (libModel.fitClaims_stdScalar)  sklearn.version: " , sklearn.__version__)
-    return sclFit
-
-
-
-def fitProviders_stdScaler(pdfData, blnIsTrain=False):
-    #--- apply scaler
-    #--- WARN:  scaling is also grouped by provider
-    from sklearn.preprocessing import StandardScaler
-
-    #--- note:  this is a numpy.ndarray
-    #--- we need to fit the scaler, and then save as a pkl file
-    #strScalerPath = m_kstrScalerPath_providers
-    #strScalerPath = m_kstrScalerPath_providers_train            
-    strScalerPath = m_kstrScalerPath_prov         
-    print("INFO (libModel.fitProviders_stdScalar):  ", strScalerPath)
-    if (blnIsTrain):
-        scaler = StandardScaler()
-        sclFit = scaler.fit(pdfData)
-        #--- if we train locally;  write out to gbc_scalar.pkl
-        #--- we do not want to overwrite the colab version used for test
-        strScalerPath = m_kstrScalerPath_providers_train       #--- works for provider training
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using local pkl for Train: ", strScalerPath)
-        with open(strScalerPath, 'wb') as filPkl:
-            pickle.dump(sclFit, filPkl)
-    else:
-        #--- we need to load the pkl file
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using colab pkl for Test: ", strScalerPath)
-        with open(strScalerPath, 'rb') as filPkl:
-            sclFit = pickle.load(filPkl)
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  sclFit.type: ", type(sclFit))
-    return sclFit
-
-
-
-def fitProviders_stdScalerSuperv(pdfData, blnIsTrain=False):
-    #--- apply scaler
-    #--- WARN:  scaling is also grouped by provider
-    from sklearn.preprocessing import StandardScaler
-
-    #--- note:  this is a numpy.ndarray
-    #--- we need to fit the scaler, and then save as a pkl file
-    strScalerPath = m_kstrScalerPath_prov
-    if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar):  ", strScalerPath)
-    if (blnIsTrain):
-        scaler = StandardScaler()
-        sclFit = scaler.fit(pdfData)
-        #--- if we train locally;  write out to gbc_scalar.pkl
-        #--- we do not want to overwrite the colab version used for test
-        strScalerPath = m_kstrBinModelPath + "stdProvider_scaler.pkl"
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using local pkl for Train: ", strScalerPath)
-        with open(strScalerPath, 'wb') as filPkl:
-            pickle.dump(sclFit, filPkl)
-    else:
-        #--- we need to load the pkl file
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  Using colab pkl for Test: ", strScalerPath)
-        with open(strScalerPath, 'rb') as filPkl:
-            sclFit = pickle.load(filPkl)
-        if (m_blnTraceOn):  print("TRACE (libModel.fitProviders_stdScalar)  sclFit.type: ", type(sclFit))
-    return sclFit
-
-
-
-def fitProviders_txfStdScaler(pdfData, blnIsTrain=False):
-    from sklearn.preprocessing import StandardScaler
-    sclFit = fitProviders_stdScaler(pdfData, blnIsTrain)
-    X_std = sclFit.transform(pdfData)
-    return X_std
-
-
-
-def fitClaims_txfStdScaler(pdfData, blnIsTrain=False):
-    from sklearn.preprocessing import StandardScaler
-    sclFit = fitClaims_stdScaler(pdfData, blnIsTrain)
-
-
-    X_std = sclFit.transform(pdfData)
-    return X_std

lib/models/mdl_xgb.py

@@ -1,66 +0,0 @@
-import pandas as pd
-from sklearn.ensemble import GradientBoostingClassifier
-import lib.utils as libPaths
-import pickle
-import sys
-
-
-m_kstrFile = __file__
-m_kstrDataPath = libPaths.pth_data
-m_kstrBinModelPath = libPaths.pth_binModels
-m_kstrModelPath_gbc = m_kstrBinModelPath + 'gbc_model_colab.pkl'
-m_kstrModelPath_prov111 = m_kstrBinModelPath + 'prov_gbc_v1.1.1_32cols.pkl'            #--- ERROR:  __randomstate_ctor() takes from 0 to 1 positional arguments but 2 were given
-m_kstrModelPath_prov121 = m_kstrBinModelPath + 'prov_gbc_v1.2.1_32cols.pkl'
-m_kstrModelPath_prov_py3816_sk111hp = m_kstrBinModelPath + 'prov_gbc_py3816_sk111hp_32cols.pkl'
-m_kstrModelPath = m_kstrModelPath_prov_py3816_sk111hp
-
-m_blnTraceOn = True
-
-
-
-#--- Supervised:  xg boost;  gradient boosting classifier
-def load_fromPkl():
-    try:
-        with open(m_kstrModelPath, 'rb') as filPkl:
-            mdlAnoms = pickle.load(filPkl)
-        return mdlAnoms
-
-    except:
-        e = sys.exc_info()
-        print("ERROR (mdl_xgb.load_fromPkl_genError):  ", e)    
-
-
-
-def save_toPkl(mdlAnoms):
-    with open(m_kstrModelPath, 'wb') as filPkl:
-        pickle.dump(mdlAnoms, filPkl)
-    return mdlAnoms
-
-
-
-def predict(npaData):
-
-    try:
-        #--- input:  numpy.ndarray of feature eng, and scaled data 
-        mdlAnoms = load_fromPkl()
-        if (m_blnTraceOn):  print("TRACE (mdl_xgb.predict):  data loaded ... ")
-        npaPredict = mdlAnoms.predict(npaData)
-
-    except:
-        e = sys.exc_info()
-        print("ERROR (mdl_xgb.predict_genError1):  ", e)  
-
-
-    #--- AttributeError: 'GradientBoostingClassifier' object has no attribute '_loss'
-    #--- version of scikit-learn?  Monika: ?.?.? ; Iain: 1.2.0
-
-    #print("INFO (type.npaPredict):  ", type(npaPredict))
-    #if (m_blnTraceOn):  print("TRACE (mdl_xgb.predict) npaPredict.shape:  ", npaPredict.shape)
-    return npaPredict
-
-
-def train(pdfTrainData):
-    mdlAnoms = GradientBoostingClassifier()
-    mdlAnoms.fit(pdfTrainData.values)
-    save_toPkl(mdlAnoms)
-    return mdlAnoms

lib/utils.py

@@ -1,38 +0,0 @@
-#--- notes:  
-#   - this file is loaded by fastapi and streamlit, so keep it independant of those libs
-#   - all path are relative to the appl working folder:  the parent of the lib folder;  ie ..\.. to this file
-
-from pathlib import Path
-
-pth_pwd = Path(__file__).resolve().parent               #--- should be \lib
-pth_appRoot = pth_pwd.parent                            #--- ..
-
-pth_root = str(pth_appRoot) + "/"
-
-pth_bin = pth_root + "bin/"
-pth_data = pth_root + "data/"
-pth_lib = pth_root + "lib/"
-pth_routes = pth_root + "routes/"
-pth_templ = pth_root + "templ/"
-pth_uix = pth_root + "uix/"
-
-#--- bin paths
-pth_binImages = pth_bin + "images/"
-pth_binModels = pth_bin + "models/"
-
-#--- data paths
-pth_dtaApp = pth_data                                   #--- working folders for app data;  for docker, should be mapped to local host mount
-pth_dtaDemoTiles = pth_data + "demo_tiles/"                   #--- dedicated area for demo data
-pth_dtaTiles = pth_data + "tiles/"
-pth_dtaWsi = pth_data + "wsi/"
-pth_dtaTileSamples = pth_dtaDemoTiles + "raw/sample/"
-
-#--- lib paths
-pth_libModels = pth_lib + "models/"
-
-#--- route paths
-pth_rteApi = pth_routes + "api/"
-pth_rteQa = pth_routes + "qa/"
-
-m_klngMaxRecords = 100
-m_klngSampleSize = 25

main.py

@@ -1,92 +0,0 @@
-'''
-    purpose:     fastAPI routing 
-'''
-
-from fastapi import FastAPI
-from fastapi.responses import HTMLResponse
-from fastapi import APIRouter, Request, Response
-from fastapi.templating import Jinja2Templates
-import uvicorn
-
-#--- import custom libraries
-import lib.utils as libUtils
-
-
-#--- imported route handlers
-from routes.api.rte_api import rteApi
-from routes.api.rte_wsi import rteWsi
-from routes.api.rte_tiles import rteTiles
-
-
-#--- fastAPI self doc descriptors
-description = """
-    Omdena Saudi Arabia:  Liver Cancer HCC Diagnosis with XAI
-    
-    <insert purpose>
-
-    ## key business benefit #1
-    ## key business benefit #2
-    ## key business benefit #3
-
-    You will be able to:
-    * key feature #1
-    * key feature #2
-    * key feature #3
-"""
-
-app = FastAPI(
-    title="App:  Omdena Saudi Arabia - Liver Cancer HCC Diagnosis with XAI",
-    description=description,
-    version="0.0.1",
-    terms_of_service="http://example.com/terms/",
-    contact={
-        "name": "Iain McKone",
-        "email": "iain.mckone@gmail.com",
-    },
-    license_info={
-        "name": "Apache 2.0",
-        "url": "https://www.apache.org/licenses/LICENSE-2.0.html",
-    },
-)
-
-
-#--- configure route handlers
-app.include_router(rteWsi, prefix="/api/wsi")
-app.include_router(rteTiles, prefix="/api/tiles")
-app.include_router(rteApi, prefix="/api")
-
-#app.include_router(rteQa, prefix="/qa")
-
-
-m_kstrPath_templ = libUtils.pth_templ
-m_templRef = Jinja2Templates(directory=str(m_kstrPath_templ))
-
-
-def get_jinja2Templ(request: Request, pdfResults, strParamTitle, lngNumRecords, blnIsTrain=False, blnIsSample=False):
-    lngNumRecords = min(lngNumRecords, libUtils.m_klngMaxRecords)
-    if (blnIsTrain):  strParamTitle = strParamTitle + " - Training Data"
-    if (not blnIsTrain):  strParamTitle = strParamTitle + " - Test Data"
-    if (blnIsSample):  lngNumRecords = libUtils.m_klngSampleSize
-    strParamTitle = strParamTitle + " - max " + str(lngNumRecords) + " rows"
-
-    kstrTempl = 'templ_showDataframe.html'
-    jsonContext = {'request': request, 
-                'paramTitle': strParamTitle,
-                'paramDataframe': pdfResults.sample(lngNumRecords).to_html(classes='table table-striped')
-            }
-    result = m_templRef.TemplateResponse(kstrTempl, jsonContext)
-    return result
-
-
-#--- get main ui/ux entry point
-@app.get('/')
-def index():
-    return {
-        "message": "Landing page:  Omdena Saudi Arabia - Liver HCC Diagnosis with XAI"
-    }
-
-
-
-if __name__ == '__main__':
-    uvicorn.run("main:app", host="0.0.0.0", port=49300, reload=True)
-#CMD ["uvicorn", "main:app", "--host=0.0.0.0", "--reload"]

packages.txt

@@ -1,3 +0,0 @@
-ffmpeg
-libsm6
-libxext6

requirements.txt

@@ -1,87 +0,0 @@
-#--- 20240625:  attempt to set the minimum version based on a working v0.1.2
-#--- 20230530:  commented out all secondary packages as they were causing the huggingfaceSpace to fail 
-
-altair<=5.0.1
-anyio<=3.7.1
-attrs<=23.1.0
-#backports.zoneinfo==0.2.1
-
-#blinker==1.6.2
-#cachetools==5.3.0
-#certifi==2023.5.7
-#charset-normalizer==3.1.0
-#click==8.1.3
-
-#decorator==5.1.1
-#entrypoints==0.4
-
-fastapi<=0.95.2
-
-#gitdb==4.0.10
-#git-lfs
-
-#GitPython==3.1.31
-grad-cam<=1.4.8
-#h11==0.14.0
-#idna==3.4
-#importlib-metadata==6.6.0
-#importlib-resources==5.12.0
-#ipython-genutils==0.2.0
-
-Jinja2<=3.1.2
-joblib<=1.2.0
-jsonschema<=4.17.3
-
-#markdown-it-py==2.2.0
-#MarkupSafe==2.1.2
-
-matplotlib<=3.7.2
-
-#mdurl==0.1.2
-
-numpy<=1.24.3
-#packaging==23.1
-
-pandas<=1.5.3
-#Pillow==9.5.0
-pip<=22.3.1
-#pkgutil_resolve_name==1.3.10
-
-plotly<=5.14.1
-#protobuf==4.23.3
-#pyarrow==12.0.1
-#pydantic==1.10.11
-#pydeck==0.8.1b0
-#Pygments==2.15.1
-#Pympler==1.0.1
-#pyrsistent==0.19.3
-#python-dateutil==2.8.2
-#pytz==2023.3
-#PyYAML==6.0
-#requests==2.31.0
-#rich==13.4.2
-
-scikit-learn==1.1.1
-#scipy<=1.11.1
-#six<=1.16.0
-#smmap==5.0.0
-#sniffio==1.3.0
-#starlette==0.27.0
-streamlit==1.24.0
-
-#tenacity==8.2.2
-#threadpoolctl==3.1.0
-#toml==0.10.2
-#toolz==0.12.0
-
-torch<=2.0.1
-torchvision<=0.15.2
-#tornado==6.3.2
-
-#typing_extensions==4.6.2
-#tzlocal==5.0.1
-#urllib3==2.0.2
-uvicorn<=0.22.0
-#validators==0.20.0
-#watchdog==3.0.0
-#zipp==3.15.0

routes/api/rte_api.py

@@ -1,79 +0,0 @@
-from fastapi import APIRouter, Request, Response
-from fastapi.responses import JSONResponse
-
-import pandas as pd
-import json
-
-#import lib.claims as libClaims
-#from lib.models import mdl_utils, mdl_xgb
-
-
-rteApi = APIRouter()
-
-
-#--- 
-@rteApi.get('/')
-def api_entry():
-    return {
-        "message": "api routing - welcome to Omdena Saudi HCC api"
-    }
-
-
-
-'''
-#--- >>> SAMPLE CODE BELOW
-#--- return json for claims data (merged)
-#--- note:  current is kaggle, but future could include from yyyymm filter
-@rteApi.get('/claims', response_class = JSONResponse)
-def api_getClaims(request: Request, response: Response):
-    pdfClaims = libClaims.load_claims()
-    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
-    result = json.loads(jsonSample)
-    return result
-
-
-#--- return json for featEng
-@rteApi.get('/claims/doFeatEng/', response_class = JSONResponse)
-def tst_claims_featEng():
-    pdfClaims = libClaims.load_claims()
-    pdfFeatEng = libClaims.do_featEng(pdfClaims)
-    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
-    result = json.loads(jsonSample)
-    return result
-
-
-@rteApi.get('/claims/doStdScaling/', response_class = JSONResponse)
-def tst_claims_stdScaling():
-    pdfClaims = libClaims.load_claims()
-    pdfFeatEng = libClaims.do_featEng(pdfClaims)
-    pdfScaled = mdl_utils.doClaims_stdScaler_toPdf(pdfFeatEng)
-
-    jsonSample = pdfClaims.head(50).to_json(orient="records", indent=4)
-    result = json.loads(jsonSample)
-    return result
-
-
-@rteApi.get('/claims/predict/superv', response_class = JSONResponse)
-@rteApi.get('/claims/predict/xgb', response_class = JSONResponse)
-def predict_xgb():
-    #--- load test data
-    pdfClaims = libClaims.load_claims()
-    pdfFeatEng = libClaims.do_featEng(pdfClaims)
-
-    npaScaled = mdl_utils.do_stdScaler(pdfFeatEng)
-    pdfScaled = mdl_utils.do_stdScaler_toPdf(npaScaled)
-  
-    ndaPredict = mdl_xgb.predict(npaScaled)
-    pdfPredict = pd.DataFrame(ndaPredict)
-
-    #--- stitch the grouped data with the labels
-    pdfResults = pdfScaled.copy()
-    pdfResults.insert(0, "hasAnom?", pdfPredict[0])
-
-    #--- filter to only those rows that are flagged with an anomaly
-    pdfResults = pdfResults[pdfResults['hasAnom?'] > 0] 
-
-    jsonSample = pdfResults.head(50).to_json(orient="records", indent=4)
-    result = json.loads(jsonSample)
-    return result
-'''

routes/api/rte_tiles.py

@@ -1,198 +0,0 @@
-from fastapi import APIRouter, Request, Response
-from fastapi.responses import HTMLResponse
-import numpy as np
-import cv2
-import os
-
-
-import main as libMain
-from lib import utils as libUtils
-
-
-m_kstrFile = __file__
-m_blnTraceOn = True
-
-m_kstrPath_templ = libUtils.pth_templ
-
-
-rteTiles = APIRouter()
-
-
-#--- 
-@rteTiles.get('/')
-def api_tiles():
-    return {
-        "message": "tiles api endpoint - welcome to the endpoint for tile image processing"
-    }
-
-
-#--- 
-@rteTiles.get('/raw/upload')
-def api_tilesRawUpload():
-    '''
-        process an array of uploaded raw Tiles (from external app path)
-        - cleanup all old raw images in /data/tiles/raw
-        - save uploads to /data/tiles/raw
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - create array of tile class objs
-        - return(s) json
-            - ack tile/raw uploads with info/attribs
-    '''
-    return {
-        "message": "tilesRawUpload endpoint - file processing of raw tile images"
-    }
-
-
-@rteTiles.get('/raw/norm')
-def api_tilesRawNormalize(strPthTile):
-    '''
-        process an array of uploaded raw Tiles (from internal app path)
-        - cleanup all old norm images in /data/tiles/norm
-        - process tile normalization ops
-        - save norm tiles to /data/tiles/norm
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - return(s) json
-            - ack tile/norms with info/attribs
-    '''
-    #--- get file attributes
-    strFilPath, strFilName = os.path.split(strPthTile)
-    strPthRaw = strPthTile
-
-    #--- load the tile as a binary object
-    with open(strPthRaw,"rb") as filRaw: 
-        imgRaw = filRaw.read()
-
-    #--- Resize Tiles to 256x256
-    #--- Note:  imgTile is a buffer object.  
-    aryNp = np.frombuffer(imgRaw, np.uint8)
-    imgTemp = cv2.imdecode(aryNp, cv2.IMREAD_COLOR)
-    imgResized = cv2.resize(imgTemp, (256, 256))
-
-    #--- save the normalized file 
-    imgNorm = imgResized
-    strPthNorm = "data/tiles/norm", strFilName
-    with open(os.path.join(strPthNorm),"wb") as filNorm: 
-        filNorm.write(imgResized.buffer)
-    return strPthNorm
-    """ return {
-        "message": "tileRawNorm endpoint - normalization of raw tile images"
-    }
- """
-
-@rteTiles.get('/norm/upload')
-def api_tilesNormUpload():
-    '''
-        process an array of uploaded norm Tiles (from external app path)
-        - cleanup all old norm images in /data/tiles/norm
-        - save uploads to /data/tiles/norm
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - create array of tile class objs
-        - return(s) json
-            - ack tile/norm uploads with info/attribs
-    '''
-    return {
-        "message": "tilesNormUpload endpoint - file processing of norm tile images"
-    }
-
-
-@rteTiles.get('/norm/preprocess')
-def api_tilesNormPreprocess():
-    '''
-        preprocess an array of uploaded norm Tiles (from internal app path)
-        - perform remaining pre-processing of tiles prior to model prediction
-        - cleanup all old preproc images in /data/tiles/preproc
-        - save preproc tiles to /data/tiles/preproc
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - return(s) json
-            - ack tile/preproc with info/attribs
-    '''
-    return {
-        "message": "tileNormPreprocess endpoint - preprocessing of normalized tile images"
-    }
-
-
-@rteTiles.get('/preproc/upload')
-def api_tilesPreprocUpload():
-    '''
-        process an array of uploaded preprocessed Tiles (from external app path)
-        - cleanup all old preproc images in /data/tiles/preproc
-        - save uploads to /data/tiles/preproc
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - create array of tile class objs
-        - return(s) json
-            - ack tile/preproc uploads with info/attribs
-    '''
-    return {
-        "message": "tilesPreprocUpload endpoint - manage upload of preprocessed tile images, in prep for modelling/prdictions"
-    }
-
-
-@rteTiles.get('/preproc/augment')
-def api_tilesPreprocAugment():
-    '''
-        process an array of uploaded preprocessed tiles (from internal app path)
-        - cleanup all old augmented tiles in /data/tiles/augm
-        - perform augments of tiles prior to model prediction (translation, rotation, transforms)
-        - save augmented tiles to /data/tiles/augm
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - return(s) json
-            - ack tile/augm with info/attribs
-    '''
-    return {
-        "message": "tilePreprocAugment endpoint - augment tile images"
-    }
-
-
-@rteTiles.get('/augm/upload')
-def api_tilesAugmUpload():
-    '''
-        process an array of augmented tiles (from external app path)
-        - cleanup all old augm images in /data/tiles/augm
-        - save uploads to /data/tiles/augm
-        - create tile class obj;  capture file path, size, zoomMagnif, etc
-        - create array of tile class objs
-        - return(s) json
-            - ack tile/augm uploads with info/attribs
-    '''
-    return {
-        "message": "tilesAugmUpload endpoint - manage upload of augmented tile images, in prep for modelling/predictions"
-    }
-
-
-#--- 
-@rteTiles.get('/raw/predict')
-def api_tileRawPredict():
-    return {
-        "message": "tile_rawPredict api endpoint - welcome to the endpoint for tile predictions"
-    }
-
-
-#--- 
-@rteTiles.get('/norm/segment')
-def api_tileNormPredict():
-    return {
-        "message": "tile_normPredict api endpoint - welcome to the endpoint for tile predictions"
-    }
-
-#--- 
-@rteTiles.get('/norm/predict')
-def api_tileNormPredict():
-    return {
-        "message": "tile_normPredict api endpoint - welcome to the endpoint for tile predictions"
-    }
-
-
-#--- 
-@rteTiles.get('/preproc/predict')
-def api_tilePreprocPredict():
-    return {
-        "message": "tile_preprocPredict api endpoint - welcome to the endpoint for tile predictions"
-    }
-
-
-#--- 
-@rteTiles.get('/augm/predict')
-def api_tileAugmPredict():
-    return {
-        "message": "tile_augmPredict api endpoint - welcome to the endpoint for tile predictions"
-    }

routes/api/rte_wsi.py

@@ -1,56 +0,0 @@
-from fastapi import APIRouter, Request, Response
-from fastapi.responses import HTMLResponse
-
-
-import main as libMain
-from lib import utils as libUtils
-
-
-m_kstrFile = __file__
-m_blnTraceOn = True
-
-m_kstrPath_templ = libUtils.pth_templ
-
-
-rteWsi = APIRouter()
-
-
-#--- 
-@rteWsi.get('/')
-def api_wsi():
-    return {
-        "message": "wsi api endpoint - welcome to the endpoint for wsi image processing"
-    }
-
-
-#--- 
-@rteWsi.get('/upload')
-def api_wsiUpload():
-    '''
-        process a single uploaded WSI image (from external app path)
-        - cleanup all old WSI images in /data/wsi/raw
-        - save upload to /data/wsi/raw
-        - create wsi class obj;  capture file path, size, zoomMagnif, etc
-        - return(s) json
-            - ack wsi upload with info/attribs
-    '''
-    return {
-        "message": "wsiUpload endpoint - file processing of one uploaded wsi image"
-    }
-
-
-#--- 
-@rteWsi.get('/chunk')
-def api_wsiChunk():
-    '''
-        process a single WSI image  (from internal app path)
-        - create wsi class obj;  capture file path, size, zoomMagnif, etc
-        - kick off tile chunking process;  
-        - save tiles to /data/tiles/raw
-        - return(s) json
-            - ack wsi upload with info/attribs
-            - ack of tiles created:  total count;  names, paths, attribs (dimensions)
-    '''
-    return {
-        "message": "wsiLoad endpoint - for chunking of wsi image to one or more tiles"
-    }

routes/qa/rte_qa.py

@@ -1,17 +0,0 @@
-from fastapi import APIRouter
-
-
-m_kstrFile = __file__
-m_blnTraceOn = True
-
-
-rteQa = APIRouter()
-
-
-@rteQa.get('/')
-@rteQa.get('/verif')
-@rteQa.get('/valid')
-def qa_entry():
-    return {
-        "message": "qa routing - welcome to Omdena Saudi HCC qa"
-    }

scripts/docker/util_docker_preRun.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-
-#--- Note:  this file is designed to run locally as well as within docker to prep the environment
-#--- Entry:  this script is assumed to run from the /app root folder
-#--- Usage:  ./scripts/docker/util_docker_preRun.sh
-
-#--- for volume initialization; ensure folders are in place; assume:  we are in the /app folder
-
-
-<<blockComment 
-- the binary model is stored as split files named mdl_nn
-- this is done to ensure that the model can be stored within gitHub
-- the split model is recreated on docker container startup using the cat command
-blockComment
-echo -e "INFO(util_docker_preRun):\t Initializing ..."
-
-strpth_pwd=$(pwd)
-strpth_scriptLoc=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-strpth_scrHome="${strpth_scriptLoc}/../"
-strpth_appHome="${strpth_scrHome}../"
-strpth_scrModels="${strpth_scrHome}models/"
-
-echo "strpth_appHome = ${strpth_appHome}"
-
-#--- recreate single model file from its parts, stored within a specific model version folder
-strpth_binModels="${strpth_appHome}bin/models/"
-echo "strpth_binModels = ${strpth_binModels}"
-#$("'${strpth_scrModels}util_joinModel.sh' '${strpth_binModels}deeplabv3*vhflip30/model_a*' '${strpth_binModels}model.pth'")
-#---eval "'${strpth_scrModels}/util_joinModel.sh' '${strpth_binModels}/deeplabv3*vhflip30/model_a*' '${strpth_binModels}/model.pth'"
-
-#--- run streamlit/fastapi
-eval "'${strpth_scrHome}/streamlitFastApi/util_local_runStreamlitFastApi.sh'"

scripts/docker/util_local_buildDockerDemo.sh

@@ -1,86 +0,0 @@
-#!/bin/bash
-
-#--- Note:  this file is designed to run locally to build the docker image
-#--- Entry:  this script is assumed to run from the /app root folder
-#--- Usage:  ./scripts/docker/util_local_buildDockerDemo.sh
-#--- Assume:  any associated containers are not running
-
-<<blockComment 
-    util_local_buildDockerDemo -> img_stm_omdenasaudi_hcc:demo -> ctr_stm_omdenasaudi_hcc:demo
-blockComment
-
-
-#--- initialize/configuration
-echo "TRACE:  Initializing ..."
-kstr_defDkrHubId="kidcoconut73"
-kstr_defDkrImageName="img_stm_omdenasaudi_hcc"
-kstr_defDkrTagVersion="0.1.3"
-kstr_defDkrTagStage="demo"
-
-strpth_pwd=$(pwd)
-strpth_scriptLoc=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-strpth_scrHome="${strpth_scriptLoc}/../"
-strpth_appHome="${strpth_scrHome}/../"
-
-#--- declarations
-echo "TRACE:  Declarations ..."
-
-#strUtl_scriptLoc="$(utl_getScriptLoc)"
-source ${strpth_scrHome}/util.sh
-
-#kstr_dkrImg="kidcoconut73/img_stm_omdenasaudi_hcc:demo"
-#kstr_dkrCtr="kidcoconut73/ctr_stm_omdenasaudi_hcc:demo"
-kstr_dkrHubImg="${kstr_defDkrHubId}/${kstr_defDkrImageName}:${kstr_defDkrTagStage}"
-kstr_dkrImg="${kstr_defDkrImageName}:${kstr_defDkrTagVersion}"
-kstr_dkrCtr="${kstr_dkrImg/img_/ctr_}"          #--- bash replace one occurrence
-
-
-
-function utl_trace_config {
-    #echo ""
-    local kbln_enableLog=true
-    utl_logInfo $kbln_enableLog 0 "(utl_trace_config) ... echo configuration"
-
-    utl_trace_var "strpth_pwd" $strpth_pwd
-    utl_trace_var "strpth_scriptLoc" $strpth_scriptLoc
-    echo ""
-    utl_trace_var "kstr_defDkrHubId" $kstr_defDkrHubId
-    utl_trace_var "kstr_defDkrImageName" $kstr_defDkrImageName
-    utl_trace_var "kstr_defDkrTagVersion" $kstr_defDkrTagVersion
-    utl_trace_var "kstr_defDkrTagStage" $kstr_defDkrTagStage
-    echo ""
-    utl_trace_var "kstr_dkrHubImg" $kstr_dkrHubImg
-    utl_trace_var "kstr_dkrImg" $kstr_dkrImg
-    utl_trace_var "kstr_dkrCtr" $kstr_dkrCtr
-    echo ""
-}
-
-#echo -e "\nTRACE:  Echo config ..."
-utl_trace_config
-
-
-#--- to build/rebuild the image;  make sure you stop and remove the container if you are replacing/upgrading;  or change the version tag# from 0.1
-#--- stop the container if it is running
-#--- delete container if it exists
-echo -e "\nTRACE:  Stop and remove container if it exists ..."
-docker stop $kstr_dkrCtr
-docker rm $kstr_dkrCtr
-
-#--- build the docker image
-echo -e "\nTRACE:  Build the docker image ..."
-docker build -t $kstr_dkrImg .
-
-
-#--- to tag the image prior to push to DockerHub;  docker login and then register user/image:tag
-#--- to push this image to DockerHub, example based on the repo:  kidcoconut73/img_stm_omdenasaudi_hcc
-#    docker tag img_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:demo
-#    docker tag img_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:0.1
-#--- tag the image
-echo -e "\nTRACE:  Tag the image ..."
-docker tag ${kstr_dkrImg} $kstr_dkrHubImg
-docker tag ${kstr_dkrImg} "${kstr_defDkrHubId}/${kstr_defDkrImageName}:${kstr_defDkrTagVersion}"
-
-
-#--- prereq:  make sure you are signed in through Docker Desktop
-#--- push the image to dockerHub
-docker push kidcoconut73/img_stm_omdenasaudi_hcc:demo

scripts/docker/util_local_runDockerDemo.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-#--- Note:  this file is designed to run locally to launch docker
-#--- Entry:  this script is assumed to run from the /app root folder
-#--- Usage:  ./scripts/util_local_runDockerDemo.sh
-#--- Assume:  docker image has been built;  container is not running
-
-<<blockComment 
-    util_local_runDockerDemo -> Dockerfile -> util_dockerPreRun -> util_local_runStreamlitFastApi
-blockComment
-
-
-#--- initialize/config
-kstr_defDkrHubId="kidcoconut73"
-kstr_defDkrImageName="img_stm_omdenasaudi_hcc"
-kstr_defDkrCtrName=${kstr_defDkrImageName/img_/ctr_}
-kstr_defDkrTagVersion="0.1.3"
-kstr_defDkrTagStage="demo"
-
-kstr_dkrImg="${kstr_defDkrImageName}:${kstr_defDkrTagVersion}"
-kstr_dkrCtr="${kstr_defDkrImageName/img_/ctr_}"          #--- bash replace one occurrence
-
-#--- stop the container if it is running
-docker stop $kstr_dkrCtr
-
-#--- delete container if it exists
-docker rm $kstr_dkrCtr
-
-#--- to run the container from the image;  specific port mapping (-p) vs any available port mapping (-P)
-#    docker run -p 49400:39400 -p 49500:39500 --name ctr_stmOmdenaSaudiHcc -v ./data:/app/data img_stm_omdenasaudi_hcc:0.1
-
-#--- run docker demo locally
-docker run -p 49400:39400 -p 49500:39500 --name $kstr_dkrCtr -v ./data:/app/data $kstr_dkrImg

scripts/huggingface/util_local_deploy_toHugSpace.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#--- Note:  this file is designed to run locally to ready the deploy branch for hugspace
+#--- Entry:  this script is assumed to run from the /app root folder
+#--- Usage:  ./scripts/huggingface/util_local_deploy_toHugSpace.sh
+
+#--- Prereqs:  
+#   - deploy_dockerLocal branch image and container has been tested and tagged
+#   - deploy_dockerHub branch has been merged from deploy_dockerLocal, tagged and pushed an image to kidcoconut73/img_stm_omdenasaudi_hcc:<version>
+#   - deploy_hugspace branch has been merged from deploy_dockerLocal, readyToDeployScript run, and pushed to hugspace
+
+<<blockComment 
+    - if all code is within a dockerHub image, all we need to push to huggingFace is the Dockerfile and the README.md
+blockComment
+
+
+#--- initialize/configuration
+echo "WARN:  this script should only be run from deploy_hugspace"
+
+
+#--- merge
+echo "TRACE:  Merge from deploy_dockerLocal ..."
+git merge deploy_dockerLocal
+
+#--- ready the Dockerfile.hug as the active Dockerfile
+echo "TRACE:  update Dockerfile for hugspace ..."
+cp Dockerfile.hug Dockerfile
+
+#--- cleanup all extraneous code
+echo "TRACE:  cleanup all extraneous code ..."
+git rm -r bin data lib routes scripts/docker scripts/dockerHub scripts/models scripts/streamlitFastApi templ uix
+git rm .dockerignore .gitattributes app.py config.toml Dockerfile.hug Dockerfile.local main.py packages.txt requirements.txt
+
+#--- manually check from this point
+#   - git status
+#   - git rm any remaining unneeded files
+#   - git commit -a -m "ready for deploy to hugspace v0.1.5"
+#   - git tag 0.1.5_hugspace
+#   - git push --force hugspace deploy_hugspace:main

scripts/huggingface/util_local_readyDeploy_toHugSpace_streamlit.sh

@@ -1,93 +0,0 @@
-#!/bin/bash
-
-#--- Note:  this file is designed to run locally to ready the deploy branch for hugspace
-#--- Entry:  this script is assumed to run from the /app root folder
-#--- Usage:  ./scripts/util_local_readyDeploy_toHugSpace_streamlit.sh
-
-#--- Prereqs:  
-#   - deploy_dockerLocal branch image and container has been tested and tagged
-#   - deploy_dockerHub branch has been merged from deploy_dockerLocal, tagged and pushed an image to kidcoconut73/img_stm_omdenasaudi_hcc:<version>
-#   - deploy_hugspace branch has been merged from deploy_dockerLocal, readyToDeployScript run, and pushed to hugspace
-
-<<blockComment 
-    bash:   util_local_readyDeploy_toHugSpace_streamlit -> git
-    git:    local/task-5-deployment -> omdena/deploy_hugspace_streamlit -> hugspace/main
-blockComment
-
-
-#--- initialize/configuration
-echo "TRACE:  Initializing ..."
-kstr_hugspaceId="kidcoconut"
-
-
-#--- git checkout deploy_hugspace_streamlit
-#--- git merge task-5-deployment
-#--- delete all unnecessary files
-<<deadCode
-    kstr_defDkrHubId="kidcoconut73"
-    kstr_defDkrImageName="img_stm_omdenasaudi_hcc"
-    kstr_defDkrTagVersion="0.1.2"
-    kstr_defDkrTagStage="demo"
-    strpth_pwd=$(pwd)
-    strpth_scriptLoc=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-
-    #--- declarations
-    echo "TRACE:  Declarations ..."
-
-    #strUtl_scriptLoc="$(utl_getScriptLoc)"
-    source ${strpth_scriptLoc}/util.sh
-
-    #kstr_dkrImg="kidcoconut73/img_stm_omdenasaudi_hcc:demo"
-    #kstr_dkrCtr="kidcoconut73/ctr_stm_omdenasaudi_hcc:demo"
-    kstr_dkrHubImg="${kstr_defDkrHubId}/${kstr_defDkrImageName}:${kstr_defDkrTagStage}"
-    kstr_dkrImg="${kstr_defDkrImageName}:${kstr_defDkrTagVersion}"
-    kstr_dkrCtr="${kstr_dkrImg/img_/ctr_}"          #--- bash replace one occurrence
-
-
-
-    function utl_trace_config () {
-        echo ""
-        utl_trace_var "strpth_pwd" $strpth_pwd
-        utl_trace_var "strpth_scriptLoc" $strpth_scriptLoc
-        echo ""
-        utl_trace_var "kstr_defDkrHubId" $kstr_defDkrHubId
-        utl_trace_var "kstr_defDkrImageName" $kstr_defDkrImageName
-        utl_trace_var "kstr_defDkrTagVersion" $kstr_defDkrTagVersion
-        utl_trace_var "kstr_defDkrTagStage" $kstr_defDkrTagStage
-        echo ""
-        utl_trace_var "kstr_dkrHubImg" $kstr_dkrHubImg
-        utl_trace_var "kstr_dkrImg" $kstr_dkrImg
-        utl_trace_var "kstr_dkrCtr" $kstr_dkrCtr
-        echo ""
-    }
-
-    echo -e "\nTRACE:  Echo config ...\n"
-    utl_trace_config
-
-
-    #--- to build/rebuild the image;  make sure you stop and remove the container if you are replacing/upgrading;  or change the version tag# from 0.1
-    #--- stop the container if it is running
-    #--- delete container if it exists
-    echo -e "\nTRACE:  Stop and remove container if it exists ..."
-    docker stop $kstr_dkrCtr
-    docker rm $kstr_dkrCtr
-
-    #--- build the docker image
-    echo -e "\nTRACE:  Build the docker image ..."
-    docker build -t $kstr_dkrImg .
-
-
-    #--- to tag the image prior to push to DockerHub;  docker login and then register user/image:tag
-    #--- to push this image to DockerHub, example based on the repo:  kidcoconut73/img_stm_omdenasaudi_hcc
-    #    docker tag img_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:demo
-    #    docker tag img_omdenasaudi_hcc:0.1 kidcoconut73/img_stm_omdenasaudi_hcc:0.1
-    #--- tag the image
-    echo -e "\nTRACE:  Tag the image ..."
-    docker tag ${kstr_dkrImg} $kstr_dkrHubImg
-    docker tag ${kstr_dkrImg} "${kstr_defDkrHubId}/${kstr_defDkrImageName}:${kstr_defDkrTagVersion}"
-
-
-    #--- push the image to dockerHub
-    #    docker push kidcoconut73/img_stm_omdenasaudi_hcc:demo
-deadCode

scripts/models/util_joinModel.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-<<blkHeader
-Name:       util_joinModel
-Purpose:    reconstitutes a split pyTorch binary model with weights, into a single binary file
-Usage:      ./util_joinModel.sh <source pattern match> <dest model file>
-            - the first arg has to be wrapped in single quotes to ensure that bash does not expand wildcards
-Prereqs:    a model folder within bin/models;  containing a split pyTorch model.pth as 1 or more model_nn files 
-Todo:       get the parent folder name and use this as the name for the model file
-blkHeader
-
-#--- dependencies
-#none
-
-
-#--- initialize/configuration
-#---    $1:  first arg;  source pattern match;  eg './bin/models/deeplabv3*vhflip30/model_a*';  Note that this is wildcarded so must be in quotes
-#---    $n:  last arg; dest model file;  eg. ./bin/models/model.pth
-echo -e "INFO(util_joinModel):\t Initializing ..."
-strPth_patternMatch=$1
-if [ -z "$strPth_patternMatch" ]; then
-    echo "WARN: no args provided.  Exiting script."
-    exit
-fi
-
-strPth_filMatch=( $strPth_patternMatch )        #--- expand the pattern match; get the first value of the pattern match
-strPth_parentFld=$(dirname $strPth_filMatch)    #--- get the parent dir of the first file match
-strPth_mdlFile=${@: -1}                         #--- Note:  this gets the last arg;  otherwise the 2nd arg would be an iteration of the 1st arg wildcard
-
-strpth_pwd=$(pwd)
-strpth_scriptLoc=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-strpth_scrHome="${strpth_scriptLoc}/../"
-strpth_appHome="${strpth_scrHome}/../"
-
-#echo "TRACE:  strPth_patternMatch= $strPth_patternMatch"
-#echo "TRACE:  strPth_filMatch= $strPth_filMatch"
-#echo "TRACE:  strPth_parentFld= $strPth_parentFld"
-#echo "TRACE:  strPth_mdlFile= $strPth_mdlFile"
-
-#--- reconstitute model
-#--- Note:  cat command does not work with single-quote literals; do not reapply single quotes
-#echo "cat ${strPth_patternMatch} > ${strPth_mdlFile}"
-echo -e "INFO:\t Joining model binary ..."
-cat ${strPth_patternMatch} > ${strPth_mdlFile}
-echo -e "INFO:\t Done ...\n"

scripts/models/util_splitModel.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-<<blkHeader
-Name:       util_splitModel
-Purpose:    convenience script to split a single pyTorch .pth model file with weights into smaller 10MB chunks in order to store within github
-Usage:      ./util_splitModel.sh <src model file> <dest folder>
-            - the first arg has to be wrapped in single quotes to ensure that bash does not expand wildcards
-Prereqs:    a pytorch model file 
-Todo:       get the parent folder name and use this as the name for the model file
-blkHeader
-
-#--- dependencies
-#none
-
-
-#---    initialization/configuration
-#---    $1:  first arg; the source model file; eg ./bin/models/model.pth
-#---    $n:  last arg; dest model path;  eg. ./test_model_folder
-strPth_mdlFile=$1
-strPth_mdlFolder=$2
-strPrefix='/model_'
-
-if [ -z "$strPth_mdlFile" ] || [ -z "$strPth_mdlFolder" ]; then
-    echo "WARN: no args provided.  Exiting script."
-    exit
-fi
-
-strpth_pwd=$(pwd)
-strpth_scriptLoc=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-strpth_scrHome="${strpth_scriptLoc}/../"
-#strpth_ignHome="${strpth_scrHome}/../"
-strpth_appHome="${strpth_scrHome}/../"
-
-#echo "TRACE:  strPth_mdlFile= $strPth_mdlFile"
-echo "TRACE:  strPth_mdlFolder= $strPth_mdlFolder"
-
-#--- ensure the target dir exists
-mkdir -p $strPth_mdlFolder
-
-#--- split the model into smaller chunks
-echo "split -b 10M $strPth_mdlFile $strPth_mdlFolder$strPrefix"
-split -b 10M $strPth_mdlFile $strPth_mdlFolder$strPrefix
-
-echo -e "INFO:\t Done ...\n"

scripts/streamlitFastApi/util_local_runStreamlitFastApi.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-#--- Note:  this file is designed to run locally and within docker to prep the environment
-#--- Entry:  this script is assumed to run from the /app root folder
-#--- Usage:  ./scripts/util_local_runStreamlitFastApi.sh
-echo -e "INFO(util_local_runStreamlitFastApi):\t Initializing ..."
-
-#--- for volume initialization; ensure folders are in place; assume:  we are in the /app folder
-mkdir -p data/demo_tiles/raw
-mkdir -p data/tiles/raw data/tiles/pred data/tiles/grad_bg data/tiles/grad_wt data/tiles/grad_vt
-mkdir -p data/wsi/raw
-
-#--- for streamlit;  external 49400;  internal 39400
-echo "INFO:  starting streamlit ..."
-streamlit run app.py --server.port=39400 --server.maxUploadSize=2000 &
-
-#--- for fastapi;  external 49500;  internal 39500
-echo "INFO:  starting fastapi ..."
-
-#--- uvicorn main:app --reload --workers 1 --host 0.0.0.0 --port 39500 &
-uvicorn main:app --reload --workers 1 --host 0.0.0.0 --port 39500 
-
-#--- wait for any process to exit
-#wait -n
-
-#--- Exit with status of process that exited first
-#exit $?