updated hosted version to v0.0.3

Dashboard.py

@@ -1,22 +1,14 @@
 import streamlit as st
 import pandas as pd 
 import numpy as np 
-
-@st.cache
-def prompt_to_csv(df):
-    # IMPORTANT: Cache the conversion to prevent computation on every rerun
-    df_download = df
-    df_download['Filename']='p'+df_download['ID'].astype('str')+'_1.png'
-    df_download = df[['Prompt','Filename']].drop_duplicates(subset='Filename')
-    return df_download.to_csv().encode('utf-8')
-
+from Dashboard_setup import prompt_dir, automated_task_list
+from pages.Functions.Dashboard_functions import prompt_to_csv
 
 # Setup
-## Load prompt directory
-prompt_dir = pd.read_csv('Data/Prompt_dir_221216.csv') #second version of prompt_dir
+## Add prompt directory to session state
 st.session_state['prompt_dir'] = prompt_dir
 ## Create lists of prompts for manual and automated assessments
-st.session_state['automated_tasks'] = ['Multiple object types', 'Single object','Negation']
+st.session_state['automated_tasks'] = automated_task_list
 automated_prompts = prompt_dir.loc[
     (prompt_dir['Auto_assessment']==True)&
     (prompt_dir['Task']).isin(st.session_state['automated_tasks'])].ID.tolist()
@@ -29,7 +21,6 @@ st.subheader('User guide')
 st.write('To assess a generative image algorithm, download a set of prompts using the prompt downloader below. Generate one image per prompt and use the file names provided to name your images. Upload these generated images in the data upload section below. The pages for manual assessment and automated assessment allow you to systematically assess the generated images. The results will be presented and ready for download on the assessment summary page.')
 st.sidebar.image('Graphics/IL_Logo.png')
 
-
 # Add prompt downloading functions
 prompt_download_dict = {}
 ## Count how many prompts are in database to allow for max value in selection
@@ -73,7 +64,6 @@ with st.expander("Prompt downloader"):
     )
 
 
-
 # Generate empty dataset for results, if it does not exist yet
 try:
     num_uploaded_images = st.session_state['eval_df'].shape[0]
@@ -88,6 +78,7 @@ try:
 except KeyError:
     st.session_state['results_dict'] = {}
 
+
 # Data upload setup
 st.subheader('Data upload')
 #uploaded_files = st.file_uploader('Upload generated images', accept_multiple_files=True)
@@ -102,6 +93,7 @@ with st.form("my-form", clear_on_submit=True):
         st.session_state['uploaded_img'] = st.session_state['uploaded_img']+uploaded_files
 
 
+
 # Add new uploaded images to session state
 ## Try to append it to pre-existing list, else create new list in session state
 ## Always reset uploaded files to empty list after they have been added to state
@@ -150,7 +142,3 @@ if eval_df.shape[0]!=0:
     st.write("- Available for automated assessment: ", str(sum(eval_df.automated_eval)))
 else:
     st.write("Upload files to start the assessment.")
-
-#st.write(eval_df)
-#st.write(prompt_dir)
-#st.session_state['eval_df']

Dashboard_automation_setup.py

@@ -0,0 +1,7 @@
+from pages.Functions.Assessment_functions import CLIP_single_object_classifier, CLIP_multi_object_recognition_DSwrapper, CLIP_object_negation, DETR_multi_object_counting_DSwrapper
+
+# Create dictionary to hold functions
+fun_dict = {
+    'Multiple object types':CLIP_multi_object_recognition_DSwrapper, 
+    'Single object':CLIP_single_object_classifier,
+    'Negation':CLIP_object_negation}

Dashboard_setup.py

@@ -0,0 +1,4 @@
+import pandas as pd 
+
+automated_task_list = ['Multiple object types', 'Single object','Negation']
+prompt_dir = pd.read_csv('Data/Prompt_dir_221216.csv')

pages/1_⚙️Manual assessment.py

@@ -2,45 +2,7 @@ import streamlit as st
 import numpy as np
 import pandas as pd
 from PIL import Image
-
-def add_previous_manual_assessments():
-    '''
-    This is a routine to allow the user to upload prior manual ratings and override
-    current ratings. This way the user can restart a manual assessment.
-    '''
-    # Create dict to translate uploaded score into str format used during manual assessment
-    Bool_str_dict = {True:'Yes',False:'No'}
-
-    st.subheader('Add previous assessments')
-    st.write('Upload results of previous assessment (as downloaded from summary page) to add these results and skip these images in your current manual assessment. Note that you can only add results for images which you have uploaded using the same file name.')
-
-    uploaded_ratings = st.file_uploader('Select .csv for upload', accept_multiple_files=False)
-    if uploaded_ratings != None:
-        try:
-            uploaded_ratings_df = pd.read_csv(uploaded_ratings)
-            overlapping_files_df =pd.merge(st.session_state['eval_df'],uploaded_ratings_df,on='File_name',how='inner')
-            st.write('Number of matching file names found: '+ str(len(overlapping_files_df)))
-            st.write('Click "Add results" button to add / override current ratings with uploaded ratings.')
-        except UnicodeDecodeError:
-            st.write('WARNING: The uploaded file has to be a .csv downloaded from the "Assessment summary" page.')
-
-
-    submitted = st.button("Add results")
-    if submitted:
-        try:
-            for row in uploaded_ratings_df.itertuples():
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval']=True
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_completed']=True
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_task_score']=Bool_str_dict[row.Score]
-
-            # Reset page after ratings were submitted
-            st.experimental_rerun()
-        except NameError:
-            st.write('You need to upload a .csv file before you can add results.')
-
+from pages.Functions.Dashboard_functions import add_previous_manual_assessments
 
 st.title('Manual assessment')
 st.write('On this page you can rate all uploaded images with regards to how good they match their respective prompts. You can see the outcome of your assessment on the summary page.')
@@ -172,6 +134,7 @@ if manual_eval_available > 0:
             # Reset page after ratings were submitted
             st.experimental_rerun()
 
+    # Add option to add previous manual assessments
     add_previous_manual_assessments()
 
 # If no files are uploaded
@@ -183,8 +146,4 @@ else:
 
 
 
-#st.session_state['eval_df'].loc[curr_manual_eval,'manual_eval_completed']=True
-#st.write(st.session_state['eval_df'])
-
-
 

pages/2_🤖Automated assessment.py

@@ -2,15 +2,7 @@ import streamlit as st
 import numpy as np
 from itertools import compress
 from PIL import Image
-from pages.Functions.Assessment_functions import CLIP_single_object_classifier, CLIP_multi_object_recognition_DSwrapper, CLIP_object_negation
-
-
-# Create dictionary to hold functions
-fun_dict = {
-    'Multiple object types':CLIP_multi_object_recognition_DSwrapper, 
-    'Single object':CLIP_single_object_classifier,
-    'Negation':CLIP_object_negation}
-
+from Dashboard_automation_setup import fun_dict
 
 st.title('Automated Assessment')
 st.write('On this page you can use automated assessment algorithms to assess how good uploaded images match their respective prompts.')
@@ -30,6 +22,9 @@ try:
     temp_prompt_dir=prompt_dir[['ID','Representations','Task_specific_label']]
     temp_prompt_dir['Prompt_no']=temp_prompt_dir['ID'].astype('str')
     curr_eval_df = curr_eval_df.merge(temp_prompt_dir,on='Prompt_no')
+
+    # Check that user correctly filled out the automation setup file
+    assert list(fun_dict.keys())==st.session_state['automated_tasks'], 'Unsure that the list of automated tasks in Dashboard_setup.py is the same as the keys of the function dict in Dashboard_automation_setup.py'
 except KeyError:
     automated_eval_available = 0
 
@@ -38,38 +33,30 @@ except KeyError:
 # Else: Note to upload images for assessment
 if automated_eval_available > 0:
     
+    # Create objects to hold selections of tasks for automated assessment
+    task_list = list(fun_dict.keys())
+    task_list_len = len(task_list)
+    task_list_selected = task_list.copy()
+
     with st.form("auto_assessment_form",clear_on_submit=True):
         # Form info statment
         st.write('Select tasks to assess with the automated assessment:')
 
-        # Add selection for available categories
-        assess_multi_object = st.checkbox(
-            'Multiple object types ({0} images available)'.format(
-                len(curr_eval_df.loc[
-                    (curr_eval_df['automated_eval']==True)&
-                    (curr_eval_df['Task']=='Multiple object types')])
-            ))
-        assess_single_object = st.checkbox(
-            'Single object type ({0} images available)'.format(
-                len(curr_eval_df.loc[
-                    (curr_eval_df['automated_eval']==True)&
-                    (curr_eval_df['Task']=='Single object')])
-            ))
+        # Create list of bool selection buttons, one for every task
+        for i_task in range(task_list_len):
+            curr_task = task_list[i_task] 
+            curr_task_count = len(curr_eval_df.loc[
+                        (curr_eval_df['automated_eval']==True)&
+                        (curr_eval_df['Task']==curr_task)])
+            task_list_selected[i_task] = st.checkbox(      
+                '{0} ({1} images available)'.format(curr_task, str(curr_task_count)))
 
-        negation = st.checkbox(
-            'Negation ({0} images available)'.format(
-                len(curr_eval_df.loc[
-                    (curr_eval_df['automated_eval']==True)&
-                    (curr_eval_df['Task']=='Negation')])
-            ))
-        
         submitted = st.form_submit_button("Start automated assessment")
-        if submitted:
+        if submitted:        
             # Create list for tasks which were selected for assessment
-            selected_tasks = list(
-                compress(
-                    ['Multiple object types','Single object','Negation'], 
-                    [assess_multi_object,assess_single_object,negation]))
+            selected_tasks = list(compress(task_list,task_list_selected))
+
+
             # Create dataset to loop over with assessment
             assessed_df = curr_eval_df.loc[
                     (curr_eval_df['automated_eval']==True)&
@@ -88,5 +75,3 @@ if automated_eval_available > 0:
             st.write('Completed assessment. Access results on the summary page.')
 else:
     st.write('Upload files on dashboard starting page to start automated assessment.')
-
-#st.write(st.session_state['auto_eval_df'])

pages/3_📊Assessment summary.py

@@ -3,6 +3,7 @@ import pandas as pd
 import seaborn as sns
 import matplotlib.pyplot as plt
 from PIL import Image
+from pages.Functions.Dashboard_functions import plot_style_simple, plot_style_combined, print_results_tabs, pre_assessment_visualisation
 side_image = Image.open('Graphics/IL_Logo.png')
 st.sidebar.image(side_image)
 
@@ -11,89 +12,6 @@ def convert_df_to_csv(df):
   # IMPORTANT: Cache the conversion to prevent computation on every rerun
   return df[['File_name','Prompt_no','Task','Score']].to_csv().encode('utf-8')
 
-def plot_style_simple(results_df, return_table = False):
-  
-
-  eval_sum = results_df.groupby('Task')['Score'].sum()
-  eval_count = results_df.groupby('Task')['Score'].count()
-  eval_share = (eval_sum/eval_count)*100
-
-  if return_table:
-    return_series = results_df.groupby('Task')['Score'].sum()/results_df.groupby('Task')['Score'].count()*100
-    return_series = return_series.rename('Percentage correct')
-    return return_series
-
-  # Add small amount to make the bars on plot not disappear
-  eval_share = eval_share+1
-
-  fig = plt.figure(figsize=(12, 3))
-  sns.barplot(x=eval_share.index, y=eval_share.values, palette='GnBu')
-  plt.xticks(rotation=-45)
-  plt.ylabel('Percentage correct')
-  plt.xlabel(' ')
-  return fig
-
-def plot_style_combined(results_df, uploaded_df = None, return_table=False):
-  # Create joined dataframe of results and uploadd_df
-  uploaded_results_df = uploaded_df
-  manual_results_df['Model']='Current'
-  uploaded_results_df['Model']='Uploaded'
-  results_df = pd.concat([manual_results_df,uploaded_results_df])
-
-  # Create scores for plot
-  eval_sum = results_df.groupby(['Model','Task'])['Score'].sum()
-  eval_count = results_df.groupby(['Model','Task'])['Score'].count()
-  eval_share = (eval_sum/eval_count)*100
-  eval_share = eval_share.reset_index()
-
-  if return_table:
-    return_series = results_df.groupby(['Task','Model'])['Score'].sum()/results_df.groupby(['Task','Model'])['Score'].count()*100
-    return_series = return_series.rename('Percentage correct')
-    return return_series
-
-  # Add small amount to make the bars on plot not disappear
-  eval_share['Score'] = eval_share['Score']+1
-
-  # Create plot
-  fig = plt.figure(figsize=(12, 3))
-  sns.barplot(data=eval_share,x='Task',y='Score',hue='Model', palette='GnBu')
-  plt.xticks(rotation=-45)
-  plt.ylabel('Percentage correct')
-  plt.xlabel(' ')
-  return fig
-
-
-def print_results_tabs(file_upload, results_df, file_upload_df=None):
-  # Create a tab for bar chart and one for table data
-  tab1, tab2 = st.tabs(["Bar chart", "Data table"])
-  with tab1:
-    # If df was uploaded for comparison, we create comparison plot, else simple plot
-    if file_upload == None:
-      fig = plot_style_simple(results_df)
-      st.pyplot(fig)
-    else:
-      fig = plot_style_combined(results_df,file_upload_df)
-      st.pyplot(fig)
-
-  with tab2:
-    # If df was uploaded for comparison, we create comparison table, else simple table
-    if file_upload == None:
-      table = plot_style_simple(results_df, return_table=True)
-      st.write(table)
-    else:
-      table = plot_style_combined(results_df,file_upload_df, return_table=True)
-      st.write(table)
-
-
-def pre_assessment_visualisation(type_str):
-  st.write('Complete {0} assessment or upload .csv with saved {0} assessment to generate summary.'.format(type_str))
-
-  # Display file uploader
-  file_upload = st.file_uploader("Upload .csv with saved {0} assessment to plot prior results.".format(type_str))
-  if file_upload != None:
-    file_upload_df = pd.read_csv(file_upload).copy()
-    print_results_tabs(file_upload=None, results_df=file_upload_df)
-
 assessment_result_frames = {}
 
 
@@ -134,8 +52,6 @@ except KeyError:
 
 
 
-
-
 st.write(' ')
 st.header('Automated assessment')
 try:

pages/Functions/Dashboard_functions.py

@@ -0,0 +1,150 @@
+# General functions and routines used in the dashboard
+
+import streamlit as st
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from PIL import Image
+
+
+##### Dashboard main page
+def prompt_to_csv(df):
+    df_download = df
+    df_download['Filename']='p'+df_download['ID'].astype('str')+'_1.png'
+    df_download = df[['Prompt','Filename']].drop_duplicates(subset='Filename')
+    return df_download.to_csv().encode('utf-8')
+
+##### Manual assessment
+
+def add_previous_manual_assessments():
+    '''
+    This is a routine to allow the user to upload prior manual ratings and override
+    current ratings. This way the user can restart a manual assessment.
+    '''
+    # Create dict to translate uploaded score into str format used during manual assessment
+    Bool_str_dict = {True:'Yes',False:'No'}
+
+    st.subheader('Add previous assessments')
+    st.write('Upload results of previous assessment (as downloaded from summary page) to add these results and skip these images in your current manual assessment. Note that you can only add results for images which you have uploaded using the same file name.')
+
+    uploaded_ratings = st.file_uploader('Select .csv for upload', accept_multiple_files=False)
+    if uploaded_ratings != None:
+        try:
+            uploaded_ratings_df = pd.read_csv(uploaded_ratings)
+            overlapping_files_df =pd.merge(st.session_state['eval_df'],uploaded_ratings_df,on='File_name',how='inner')
+            st.write('Number of matching file names found: '+ str(len(overlapping_files_df)))
+            st.write('Click "Add results" button to add / override current ratings with uploaded ratings.')
+        except UnicodeDecodeError:
+            st.write('WARNING: The uploaded file has to be a .csv downloaded from the "Assessment summary" page.')
+
+
+    submitted = st.button("Add results")
+    if submitted:
+        try:
+            for row in uploaded_ratings_df.itertuples():
+                st.session_state['eval_df'].loc[
+                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval']=True
+                st.session_state['eval_df'].loc[
+                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_completed']=True
+                st.session_state['eval_df'].loc[
+                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_task_score']=Bool_str_dict[row.Score]
+
+            # Reset page after ratings were submitted
+            st.experimental_rerun()
+        except NameError:
+            st.write('You need to upload a .csv file before you can add results.')
+
+
+##### Assessment summary
+def plot_style_simple(results_df, return_table = False):
+    '''
+    Simple plot function for plotting just one dataframe of results
+    '''
+    eval_sum = results_df.groupby('Task')['Score'].sum()
+    eval_count = results_df.groupby('Task')['Score'].count()
+    eval_share = (eval_sum/eval_count)*100
+
+    if return_table:
+        return_series = results_df.groupby('Task')['Score'].sum()/results_df.groupby('Task')['Score'].count()*100
+        return_series = return_series.rename('Percentage correct')
+        return return_series
+
+    # Add small amount to make the bars on plot not disappear
+    eval_share = eval_share+1
+
+    fig = plt.figure(figsize=(12, 3))
+    sns.barplot(x=eval_share.index, y=eval_share.values, palette='GnBu')
+    plt.xticks(rotation=-65)
+    plt.ylabel('Percentage correct')
+    plt.xlabel(' ')
+    return fig
+
+def plot_style_combined(results_df, uploaded_df = None, return_table=False):
+    '''
+    Plot function which can plot to dataframe for comparison
+    '''
+    # Create joined dataframe of results and uploadd_df
+    uploaded_results_df = uploaded_df
+    manual_results_df['Model']='Current'
+    uploaded_results_df['Model']='Uploaded'
+    results_df = pd.concat([manual_results_df,uploaded_results_df])
+
+    # Create scores for plot
+    eval_sum = results_df.groupby(['Model','Task'])['Score'].sum()
+    eval_count = results_df.groupby(['Model','Task'])['Score'].count()
+    eval_share = (eval_sum/eval_count)*100
+    eval_share = eval_share.reset_index()
+
+    if return_table:
+        return_series = results_df.groupby(['Task','Model'])['Score'].sum()/results_df.groupby(['Task','Model'])['Score'].count()*100
+        return_series = return_series.rename('Percentage correct')
+        return return_series
+
+    # Add small amount to make the bars on plot not disappear
+    eval_share['Score'] = eval_share['Score']+1
+
+    # Create plot
+    fig = plt.figure(figsize=(12, 3))
+    sns.barplot(data=eval_share,x='Task',y='Score',hue='Model', palette='GnBu')
+    plt.xticks(rotation=-65)
+    plt.ylabel('Percentage correct')
+    plt.xlabel(' ')
+    return fig
+
+
+def print_results_tabs(file_upload, results_df, file_upload_df=None):
+    '''
+    Routine used to give user the choice between showing results as bar chart or table
+    '''
+    # Create a tab for bar chart and one for table data
+    tab1, tab2 = st.tabs(["Bar chart", "Data table"])
+    with tab1:
+        # If df was uploaded for comparison, we create comparison plot, else simple plot
+        if file_upload == None:
+            fig = plot_style_simple(results_df)
+            st.pyplot(fig)
+        else:
+            fig = plot_style_combined(results_df,file_upload_df)
+            st.pyplot(fig)
+
+    with tab2:
+        # If df was uploaded for comparison, we create comparison table, else simple table
+        if file_upload == None:
+            table = plot_style_simple(results_df, return_table=True)
+            st.write(table)
+        else:
+            table = plot_style_combined(results_df,file_upload_df, return_table=True)
+            st.write(table)
+
+
+def pre_assessment_visualisation(type_str):
+    '''
+    Routine used to allow user to visualise uploaded results before completing any assessments
+    '''
+    st.write('Complete {0} assessment or upload .csv with saved {0} assessment to generate summary.'.format(type_str))
+
+    # Display file uploader
+    file_upload = st.file_uploader("Upload .csv with saved {0} assessment to plot prior results.".format(type_str))
+    if file_upload != None:
+        file_upload_df = pd.read_csv(file_upload).copy()
+        print_results_tabs(file_upload=None, results_df=file_upload_df)