Create a streamlit app to display qualitative checks on totalsegmentator

Dockerfile

@@ -0,0 +1,39 @@
+# Use an official Python runtime as a parent image
+FROM python:3.12.3
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Try and run pip command after setting the user with `USER user` to avoid permission issues with Python
+RUN pip install --no-cache-dir --upgrade pip
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Create the .streamlit directory
+RUN mkdir -p .streamlit
+
+# Create the config.toml file and set the maxMessageSize
+RUN echo "\
+[server]\n\
+maxMessageSize = 2000\n\
+" > .streamlit/config.toml
+
+# Make port 8501 available to the world outside this container
+EXPOSE 8501
+
+# Run filter_data_app.py when the container launches
+CMD streamlit run filter_data_app.py

README.md

@@ -4,6 +4,7 @@ emoji: 🌍
 colorFrom: purple
 colorTo: green
 sdk: docker
+app_port: 8501
 pinned: false
 license: mit
 ---

filter_data_app.py

@@ -0,0 +1,164 @@
+import streamlit as st
+import duckdb
+import os
+import requests
+import pandas as pd
+from upsetplot import UpSet
+import matplotlib.pyplot as plt
+
+# Set page configuration
+st.set_page_config(layout="wide")
+
+# URL and local path to the Parquet file
+PARQUET_URL = 'https://github.com/vkt1414/idc-index-data/releases/download/0.1/qualitative_checks.parquet'
+LOCAL_PARQUET_FILE = 'qualitative_checks.parquet'
+
+# Function to download the Parquet file if it doesn't exist
+def download_parquet(url, local_path):
+    if not os.path.exists(local_path):
+        response = requests.get(url)
+        with open(local_path, 'wb') as file:
+            file.write(response.content)
+        st.write(f"Downloaded {local_path}")
+
+@st.cache_data
+def load_data():
+    return duckdb.query(f"SELECT * exclude(CT_SeriesInstanceUID, StudyInstanceUID), concat('https://viewer.imaging.datacommons.cancer.gov/viewer/',studyInstanceUID) as viewerUrl FROM read_parquet('{LOCAL_PARQUET_FILE}')").to_df()
+
+# Function to filter data based on user input
+def filter_data(df, filters):
+    for col, value in filters.items():
+        if value:
+            df = df[df[col] == value]
+    return df
+
+# Function to create an UpSet plot for failed checks
+def create_upset_plot_failures(df):
+    df = df.set_index(df['segmentation_completeness'] != 'pass').set_index(df['laterality_check'] != 'pass', append=True)
+    df = df.set_index(df['series_with_vertabra_on_every_slice'] != 'pass', append=True)
+    df = df.set_index(df['connected_volumes'] != 'pass', append=True)
+    df = df[df.index.to_frame().any(axis=1)]  # Ignore the case when all conditions are false
+
+    fig = plt.figure()
+    upset = UpSet(df, sort_by='cardinality', intersection_plot_elements=5, totals_plot_elements=4, element_size=40, show_counts=True)
+    upset.plot(fig=fig)
+    st.pyplot(fig)
+
+# Function to create an UpSet plot for passed checks
+def create_upset_plot_passes(df):
+    df = df.set_index(df['segmentation_completeness'] == 'pass').set_index(df['laterality_check'] == 'pass', append=True)
+    df = df.set_index(df['series_with_vertabra_on_every_slice'] == 'pass', append=True)
+    df = df.set_index(df['connected_volumes'] == 'pass', append=True)
+    df = df[df.index.to_frame().any(axis=1)]  # Ignore the case when all conditions are false
+
+    fig = plt.figure()
+    upset = UpSet(df, sort_by='cardinality', intersection_plot_elements=5, totals_plot_elements=4, element_size=40, show_counts=True)
+    upset.plot(fig=fig)
+    st.pyplot(fig)
+
+# Main function to run the Streamlit app
+def main():
+    st.title("Qualitative checks of TotalSegmentator Segmentations on NLST")
+    st.write("Filter the data based on qualitative checks, bodyPart, and laterality.")
+
+    # Information about the checks
+    with st.expander("About the Checks"):
+        st.write("""
+        **Check for Segmentation Completeness**: 
+        Depending on the inferior to superior extent of the patient scanned, certain anatomical structures may be cropped or incomplete. We evaluated the completeness of the segmentation by ensuring that there was at least one transverse slice superior and one inferior to each anatomical region. 
+
+        **Check for a Single Connected Component**: 
+        Each anatomical region that is segmented should be continuous and consist of a single connected component. We detect the presence of unconnected components using the VoxelNum field. This field informs the user not only of incomplete or missing segmentations but also the ability to identify segmentations with extraneous or noisy voxels. 
+
+        **Check for Correct Left vs Right Laterality**: 
+        One of the major issues with identifying the accuracy of a segmentation is to determine if the laterality of the anatomical region is correct. We evaluated the laterality by using metadata extracted from the segmentations using the CenterOfMass field. 
+
+        **Check for Presence of Vertebrae on Each Slice**: 
+        Specifically for the vertebrae, we hypothesized that the vertebrae must be present in every transverse slice of the scan. This heuristic is based on the superior to inferior direction of most chest scans.
+        """)
+
+    # Download the Parquet file if it doesn't exist
+    download_parquet(PARQUET_URL, LOCAL_PARQUET_FILE)
+
+    # Load the data
+    df = load_data()
+
+    # Preselect one combination for bodyPart based on the first row
+    initial_body_part = df.iloc[0]['bodyPart']
+
+    # Sidebar widgets for filtering
+    with st.sidebar:
+        st.title("Filters")
+
+        # Body part filter
+        body_part_options = df['bodyPart'].unique().tolist()
+        body_part = st.selectbox("Body Part", options=body_part_options, index=body_part_options.index(initial_body_part))
+
+        # Filter the dataframe based on the selected body part
+        filtered_df = df[df['bodyPart'] == body_part]
+
+        # Fetch unique values for other filters based on the filtered dataframe
+        segmentation_completeness_options = [""] + filtered_df['segmentation_completeness'].unique().tolist()
+        laterality_check_options = [""] + filtered_df['laterality_check'].unique().tolist()
+        series_with_vertabra_on_every_slice_options = [""] + filtered_df['series_with_vertabra_on_every_slice'].unique().tolist()
+        connected_volumes_options = [""] + filtered_df['connected_volumes'].unique().tolist()
+        laterality_options = [""] + filtered_df['laterality'].unique().tolist()
+
+        segmentation_completeness = st.selectbox("Segmentation Completeness", options=segmentation_completeness_options)
+        laterality_check = st.selectbox("Laterality Check", options=laterality_check_options)
+        series_with_vertabra_on_every_slice = st.selectbox("Series with Vertebra on Every Slice", options=series_with_vertabra_on_every_slice_options)
+        connected_volumes = st.selectbox("Connected Volumes", options=connected_volumes_options)
+        laterality = st.selectbox("Laterality", options=laterality_options)
+
+    # Filtering the data based on user input
+    filters = {
+        'segmentation_completeness': segmentation_completeness if segmentation_completeness else None,
+        'laterality_check': laterality_check if laterality_check else None,
+        'series_with_vertabra_on_every_slice': series_with_vertabra_on_every_slice if series_with_vertabra_on_every_slice else None,
+        'connected_volumes': connected_volumes if connected_volumes else None,
+        'bodyPart': body_part,
+        'laterality': laterality if laterality else None
+    }
+
+    filtered_df = filter_data(df, filters)
+
+    # Pagination for the filtered dataframe
+    page_size = 10
+    total_pages = (len(filtered_df) // page_size) + 1
+    page_number = st.sidebar.slider("Page Number", min_value=1, max_value=total_pages, value=1)
+    start_idx = (page_number - 1) * page_size
+    end_idx = start_idx + page_size
+    paginated_df = filtered_df.iloc[start_idx:end_idx]
+
+    # Display the paginated dataframe
+    st.header("Filtered Data")
+    st.write("Number of Rows:", len(filtered_df))
+    
+    st.data_editor(
+        paginated_df,
+        column_config={
+            "viewerUrl": st.column_config.LinkColumn("Viewer Url")
+        },
+        hide_index=True,
+    )
+
+    # Explanation about the UpSet plot
+    with st.expander("About the UpSet Plot"):
+        st.write("""
+        The UpSet plot is a way to visualize intersections of multiple sets. Each row in the plot represents a different set, and the dots indicate the presence or absence of intersections among these sets. The vertical bars show the size of each intersection, making it easy to see which intersections are most common.
+        """)
+
+    # Create and display the UpSet plot for failed checks
+    st.header("UpSet Plot for Failed Checks")
+    st.write("This plot shows the combinations of checks that failed.")
+    if not filtered_df.empty:
+        create_upset_plot_failures(filtered_df)
+
+    # Create and display the UpSet plot for passed checks
+    st.header("UpSet Plot for Passed Checks")
+    st.write("This plot shows the combinations of checks that passed.")
+    if not filtered_df.empty:
+        create_upset_plot_passes(filtered_df)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,7 @@
+duckdb
+matplotlib
+pandas
+pyarrow
+streamlit
+streamlit_extras
+upsetplot