updated to v0.0.5

Dashboard.py

@@ -1,10 +1,20 @@
 import streamlit as st
 import pandas as pd 
 import numpy as np 
-from Dashboard_setup import prompt_dir, automated_task_list
-from pages.Functions.Dashboard_functions import prompt_to_csv
+from Dashboard_setup import prompt_dir, automated_task_list, sidebar_information, compatible_versions, dashboard_version_code
+from pages.Functions.Dashboard_functions import prompt_to_csv, prompt_df_for_download
 
-# Setup
+
+# Page
+st.title('Generative Image Benchmark')
+st.write('This is an evaluation platform to assess the performance of image generation algorithms developed by Intel Labs. This is the beta version of the platform.')
+st.subheader('User guide')
+st.write('To assess a generative image algorithm, download a set of prompts using the prompt downloader below. Generate one image per prompt and use the file names provided to name your images. Upload these generated images in the data upload section below. The pages for manual assessment and automated assessment allow you to systematically assess the generated images. The results will be presented and ready for download on the assessment summary page.')
+sidebar_information()
+
+
+
+###### Setup of variables ############################
 ## Add prompt directory to session state
 st.session_state['prompt_dir'] = prompt_dir
 ## Create lists of prompts for manual and automated assessments
@@ -14,48 +24,29 @@ automated_prompts = prompt_dir.loc[
     (prompt_dir['Task']).isin(st.session_state['automated_tasks'])].ID.tolist()
 manual_prompts = prompt_dir.ID.tolist()
 
-# Page
-st.title('Generative Image Benchmark')
-st.write('This is an evaluation platform to assess the performance of image generation algorithms developed by Intel Labs. This is the alpha version of the platform.')
-st.subheader('User guide')
-st.write('To assess a generative image algorithm, download a set of prompts using the prompt downloader below. Generate one image per prompt and use the file names provided to name your images. Upload these generated images in the data upload section below. The pages for manual assessment and automated assessment allow you to systematically assess the generated images. The results will be presented and ready for download on the assessment summary page.')
-st.sidebar.image('Graphics/IL_Logo.png')
-
-# Add prompt downloading functions
-prompt_download_dict = {}
-## Count how many prompts are in database to allow for max value in selection
-prompt_task_count = prompt_dir.Task.value_counts(sort=False)
-prompt_task_count = prompt_task_count.drop(index='Single object')
-prompt_task_select = prompt_task_count.copy()
-## Hide downloader in box
+# Generate empty dataset for results, if it does not exist yet
+try:
+    num_uploaded_images = st.session_state['eval_df'].shape[0]
+except KeyError:
+    st.session_state['eval_df'] = pd.DataFrame(
+        columns=['File_name','Prompt_no','automated_eval','manual_eval','manual_eval_completed','manual_eval_task_score'])
+    st.session_state['uploaded_img'] = []
+
+# Create dic for automated asssssment if it does not excist yet
+try:
+    test_dict = st.session_state['results_dict']
+except KeyError:
+    st.session_state['results_dict'] = {}
+
+
+
+###### Prompt downloader ############################
+## Add prompt downloading routine in expander box
 with st.expander("Prompt downloader"):
     st.write('Select the number of prompts you want to download for each task category. The set of prompts will automatically also include all single objects appearing in the selected prompts.')
-    # Create numerical selector for every task in prompt directory
-    for i_task in prompt_task_select.index:
-        prompt_task_select[i_task] = st.number_input(
-            i_task,
-            value = prompt_task_count[i_task], 
-            max_value=prompt_task_count[i_task],
-            min_value=0,
-            step = 1)
-
-    # Create df with selected number of prompts per task
-    for i_task in prompt_task_select.index:
-        temp_df = prompt_dir.loc[prompt_dir['Task']==i_task][0:prompt_task_select[i_task]]
-        if len(temp_df)>0:
-            prompt_download_dict[i_task]=temp_df
-
-    # Concat all tasks to dataframe
-    prompt_download = pd.concat(prompt_download_dict.values())
-    # Exclude prompts from single object prompt download, as else the int transform gives an error
-    single_object_prompt_download = prompt_download.dropna(subset='Linked_prompts')
-
-    # Add relevant single object prompts
-    single_object_ids = single_object_prompt_download.Linked_prompts.str.split(',').explode().unique().astype('int')
-    prompt_download = pd.concat([
-       prompt_download,
-       prompt_dir.loc[prompt_dir['ID'].isin(single_object_ids)] 
-    ])
+
+    # Add elements to allow user to select count of prompts per task
+    prompt_download = prompt_df_for_download(prompt_dir)
 
     # For img2img prompt, the prompt in the download gets replaced by img2img instructions
     img2img_instructions_col = prompt_download.loc[prompt_download['Task'].str.startswith('img2img')]['img2img_instructions']
@@ -64,28 +55,15 @@ with st.expander("Prompt downloader"):
     # Add download button for prompts
     st.download_button(
         label="Download prompts",
-        data=prompt_to_csv(prompt_download),
+        data=prompt_to_csv(prompt_download, added_version_code=dashboard_version_code),
         file_name='prompt_list.csv',
         mime='text/csv',
     )
 
 
-# Generate empty dataset for results, if it does not exist yet
-try:
-    num_uploaded_images = st.session_state['eval_df'].shape[0]
-except KeyError:
-    st.session_state['eval_df'] = pd.DataFrame(
-        columns=['File_name','Prompt_no','automated_eval','manual_eval','manual_eval_completed','manual_eval_task_score'])
-    st.session_state['uploaded_img'] = []
 
-# Create dic for automated asssssment if it does not excist yet
-try:
-    test_dict = st.session_state['results_dict']
-except KeyError:
-    st.session_state['results_dict'] = {}
 
-
-# Data upload setup
+###### Data uploader and eval_df creation ############################
 st.subheader('Data upload')
 #uploaded_files = st.file_uploader('Upload generated images', accept_multiple_files=True)
 with st.form("my-form", clear_on_submit=True):
@@ -98,8 +76,6 @@ with st.form("my-form", clear_on_submit=True):
         submitted = st.form_submit_button("Add images")
         st.session_state['uploaded_img'] = st.session_state['uploaded_img']+uploaded_files
 
-
-
 # Add new uploaded images to session state
 ## Try to append it to pre-existing list, else create new list in session state
 ## Always reset uploaded files to empty list after they have been added to state
@@ -107,16 +83,24 @@ if len(uploaded_files) != 0:
     try:
         # Extract prompts of uploaded files
         file_names = [x.name for x in uploaded_files]
-        files_prompts = [x.split('_')[0][1:] for x in file_names]
+        files_prompts = [x.split('_',maxsplit=1)[0][1:] for x in file_names]
+        try:
+            files_versions = [x.split('_v',maxsplit=1)[1] for x in file_names]
+            files_compatible = [x.rsplit('.',1)[0] in compatible_versions for x in files_versions]
+        except IndexError:
+            files_compatible = [False]*len(files_prompts)
 
         # Create manual evaluation df
-        df_dict = {'File_name':file_names, 'Prompt_no':files_prompts}
+        df_dict = {'File_name':file_names, 'Prompt_no':files_prompts, 'File_compatible':files_compatible}
         eval_df = pd.DataFrame(df_dict)
         eval_df['automated_eval'] = eval_df['Prompt_no'].astype('int').isin(automated_prompts)
         eval_df['manual_eval'] = eval_df['Prompt_no'].astype('int').isin(manual_prompts)
         eval_df['manual_eval_completed'] = False
         eval_df['manual_eval_task_score'] = np.nan
 
+        # Set manual and automated eval = False if files are not compatible
+        eval_df.loc[eval_df['File_compatible']==False,['automated_eval','manual_eval']]=False
+
         # Exclude given percentage of uploaded images from manual assessment; with random selection
         if man_assessment_share == '50%':
             reassign_number = int(len(eval_df)/2)
@@ -139,6 +123,7 @@ if len(uploaded_files) != 0:
         st.session_state['uploaded_img'] = uploaded_files
 
 
+###### Upload status visualisation ############################
 eval_df = st.session_state['eval_df']
 if eval_df.shape[0]!=0:
     # Print current state of uploaded data
@@ -149,6 +134,7 @@ if eval_df.shape[0]!=0:
 
     if eval_df.shape[0]>sum(eval_df.manual_eval):
         st.write('WARNING: {0} image(s) with invalid file names uploaded. Pictures with invalid names will not be available for assessment. Use the file names provided by the prompt downloader to correctly name your generated images.'.format(str(eval_df.shape[0]-sum(eval_df.manual_eval))))
-
+    if eval_df.shape[0]>sum(eval_df.File_compatible):
+        st.write('WARNING: Some of the images uploaded are not compatible with this version of benchmark software. Please go to https://github.com/8erberg/Intel-Generative-Image-Dashboard-experimental/blob/main/README.md to learn more about hosting the version compatible with your images.')
 else:
     st.write("Upload files to start the assessment.")

Dashboard_setup.py

@@ -1,4 +1,19 @@
+import streamlit as st
 import pandas as pd 
 
+# Dashboard version variables
+code_version = 'v0.0.5'
+prompt_dir_version = '230131'
+compatible_versions = ['0.0.5_pd230118', 'None']
+dashboard_version_code = code_version+'_pd'+prompt_dir_version
+
+# List of tasks which are automated in current version - note that each of these needs a corresponding evaluation function in Dashboard_automation_setup.py
 automated_task_list = ['Multiple object types', 'Single object','Negation']
-prompt_dir = pd.read_csv('Data/Prompt_dir_230104.csv')
+
+# Import the list of prompts used in current version
+prompt_dir = pd.read_csv('data/Prompt_dir_{0}.csv'.format(prompt_dir_version))
+
+# Create sidebar information
+def sidebar_information():
+    st.sidebar.image('assets/IL_Logo.png')
+    st.sidebar.text(dashboard_version_code)

pages/1_⚙️Manual assessment.py

@@ -2,18 +2,20 @@ import streamlit as st
 import numpy as np
 import pandas as pd
 from PIL import Image
-from pages.Functions.Dashboard_functions import add_previous_manual_assessments, delete_last_manual_rating
-
+from pages.Functions.Dashboard_functions import add_previous_manual_assessments, delete_last_manual_rating, if_true_rerun, radio_rating_index_translation, set_eval_df_rating_vals, collect_linked_prompt_ratings
+from Dashboard_setup import sidebar_information, dashboard_version_code
 
 st.title('Manual assessment')
 st.write('On this page you can rate all uploaded images with regards to how good they match their respective prompts. You can see the outcome of your assessment on the summary page.')
 st.write(' ')
-side_image = Image.open('Graphics/IL_Logo.png')
-st.sidebar.image(side_image)
+sidebar_information()
 # Create placeholders for key elements
 assessment_header = st.empty()
+include_subprompts_checkbox = st.empty()
 assessment_progress = st.empty()
+assessment_progress_bar = st.empty()
 
+###### Setup of variables ############################
 # Extract how many images are available for manual assessment in entire uploaded dataset
 ## Set to zero if the dataset has not been created yet due to starting the app on an assessment page
 manual_eval_available = 0
@@ -21,6 +23,7 @@ try:
     curr_eval_df = st.session_state['eval_df']
     curr_eval_df['Picture_index']=curr_eval_df.index.values
     curr_manual_eval = curr_eval_df.loc[(curr_eval_df['manual_eval']==True)&(curr_eval_df['manual_eval_completed']==False)]
+    curr_manual_eval_max = len(curr_eval_df.loc[(curr_eval_df['manual_eval']==True)])
     manual_eval_available = len(curr_manual_eval)
     curr_prompt_dir = st.session_state['prompt_dir']
 except KeyError:
@@ -36,18 +39,18 @@ except IndexError:
     pass
 
 
-# Main rating loop
+###### Rating loop ############################
 ## If images are available for rating this creates a from to submit ratings to database
 ## If subprompt option is selected, it expands the form to include these as well
 ## If no images are available it prints situation specific instructions
 if manual_eval_available > 0:
     assessment_header.subheader('Assess uploaded images')
     # Let user choose whether subprompts should be presented
-    include_subprompts = st.checkbox('Show related subprompts if available (uploaded subprompts may not be shown if images have been assessed already).', value=True)
+    include_subprompts = include_subprompts_checkbox.checkbox('Show related subprompts if available (uploaded subprompts may not be shown if images have been assessed already).', value=True)
 
-    # Update the progress statement
+    # Update the progress statement / bar
     assessment_progress.write('{0} images ready / left for assessment.'.format(manual_eval_available))
-
+    assessment_progress_bar.progress(1-manual_eval_available/curr_manual_eval_max)
 
     # Extract first example for manual assessment which is not rated yet (first meaning the lowest index, for lowest prompt number)
     ## Also extract relevant metadata of this example
@@ -83,41 +86,25 @@ if manual_eval_available > 0:
             curr_prompt_dir.loc[curr_prompt_dir['ID']==int(curr_manual_eval_row.Prompt_no.item())]['Prompt'].item()
         ))
         # Exclude prompt from rating if user chooses to
-        include_prompt = st.checkbox('Include this prompt in assessment summary', value=True)
-        
+        exclude_prompt = st.checkbox('Exclude this prompt from manual assessment', value=False)
+        include_prompt = not exclude_prompt
+
         # Show image of current prompt and rating
         st.image(st.session_state['uploaded_img'][curr_manual_eval_row.Picture_index.item()],width=350)
+
+        # Preselected radio option
+        radio_preselect = radio_rating_index_translation(curr_manual_eval_row.manual_eval_task_score.item())
+
+        # Create rating element for main prompt
         curr_manual_eval_row['manual_eval_task_score'] = st.radio(
-                "Does the image match the prompt?",('Yes', 'No'), horizontal=True, key='base')
+                "Does the image match the prompt?",('Yes', 'No'), horizontal=True, key='base', index=radio_preselect)
 
         st.write(' ') # Create whitespace
         st.write(' ') # Create whitespace
 
-        # If there are linked prompts, create df with info
-        # Else create emtpy df which will automatically skip the rating creation for these prompts
-        # Here we do not test for (curr_eval_df['manual_eval']==True) as the curr_linked_prompts is already testing for valid prompt number and we want to ignore the exclusion for subprompts
-        if type(curr_linked_prompts)==list:
-            curr_linked_rows = curr_eval_df.loc[
-                (curr_eval_df['manual_eval_completed']==False)&
-                (curr_eval_df['Prompt_no'].isin(curr_linked_prompts))]
-            curr_linked_rows = curr_linked_rows.groupby('Prompt_no').first()
-        else:
-            curr_linked_rows = pd.DataFrame()
-
-        # Create rating for subprompts if a df for subprompt info was created
-        for row in curr_linked_rows.itertuples():
-            # Prompt
-            st.write('Prompt: {0}'.format(
-                curr_prompt_dir.loc[curr_prompt_dir['ID']==int(row.Index)]['Prompt'].item()
-            ))
-            # Image
-            st.image(st.session_state['uploaded_img'][row.Picture_index],width=350)
-            # Rating
-            curr_linked_rows.loc[curr_linked_rows['Picture_index']==row.Picture_index,'manual_eval_task_score'] = st.radio(
-                "Does the image match the prompt?",('Yes', 'No'), horizontal=True, key=row.Picture_index)
-            st.write(' ')
-            st.write(' ')
-        
+        # Create elements to collect ratings on linked prompts
+        # This only happens if the current prompt has linked prompts and the user choose to show linked prompts
+        curr_linked_rows = collect_linked_prompt_ratings(curr_linked_prompts, curr_eval_df, curr_prompt_dir)
 
         # Submit assessments to database
         submitted = st.form_submit_button("Submit")
@@ -126,12 +113,13 @@ if manual_eval_available > 0:
             temp_picture_index_list = []
 
             # First add main prompt assessment
-            st.session_state['eval_df'].loc[
-                curr_picture_index,'manual_eval']=include_prompt
-            st.session_state['eval_df'].loc[
-                curr_picture_index,'manual_eval_completed']=True
-            st.session_state['eval_df'].loc[
-                curr_picture_index,'manual_eval_task_score']=curr_manual_eval_row['manual_eval_task_score'].item()       
+            st.session_state['eval_df'] = set_eval_df_rating_vals(
+                st.session_state['eval_df'],
+                picture_index=curr_picture_index,
+                manual_eval=include_prompt,
+                manual_eval_completed=True,
+                manual_eval_task_score=curr_manual_eval_row['manual_eval_task_score'].item() 
+            )       
 
             # Add picture index to temp list
             temp_picture_index_list.append(curr_picture_index)
@@ -139,27 +127,32 @@ if manual_eval_available > 0:
             # Add subprompt assessment if dataset was created for subprompts
             # This stage will automatically be skipped if the df for linked prompts is empty
             for row in curr_linked_rows.itertuples():
-                st.session_state['eval_df'].loc[
-                    row.Picture_index,'manual_eval']=include_prompt
-                st.session_state['eval_df'].loc[
-                    row.Picture_index,'manual_eval_completed']=True
-                st.session_state['eval_df'].loc[
-                    row.Picture_index,'manual_eval_task_score']=row.manual_eval_task_score
-
+                st.session_state['eval_df'] = set_eval_df_rating_vals(
+                    st.session_state['eval_df'],
+                    picture_index=row.Picture_index,
+                    manual_eval=include_prompt,
+                    manual_eval_completed=True,
+                    manual_eval_task_score=row.manual_eval_task_score
+                )       
+  
                 # Add picture index to temp list
                 temp_picture_index_list.append(row.Picture_index)
 
-            # Add temp list of picture indices to rating history
-            st.session_state['manual_rating_history'].append(temp_picture_index_list)
+            # Add temp list of picture indices to rating history, if prompt is not excluded
+            if include_prompt:
+                st.session_state['manual_rating_history'].append(temp_picture_index_list)
 
             # Reset page after ratings were submitted
             st.experimental_rerun()
 
-    # Return to last rated image
-    delete_last_manual_rating()
+    # Allow user to return to last manual rating
+    st.session_state['manual_rating_history'],st.session_state['eval_df'], bool_rating_deleted = delete_last_manual_rating(
+        st.session_state['manual_rating_history'],st.session_state['eval_df'])
+    if_true_rerun(bool_rating_deleted)
 
-    # Add option to add previous manual assessments
-    add_previous_manual_assessments()
+    # Allow user to upload past ratings and add them to eval_df
+    st.session_state['eval_df'], bool_ratings_uploaded = add_previous_manual_assessments(st.session_state['eval_df'],dashboard_version_code=dashboard_version_code)
+    if_true_rerun(bool_ratings_uploaded)
 
 # If no files are uploaded
 elif len(st.session_state['uploaded_img'])==0:
@@ -167,6 +160,8 @@ elif len(st.session_state['uploaded_img'])==0:
 # If files are uploaded but all ratings are completed
 else:
     assessment_progress.write('You finished assessing the current batch of uploaded images. Upload more pictures of generate your results on the summary page.')
-    # Add option to return to last manual rating
-    delete_last_manual_rating()
 
+    # Allow user to return to last manual rating
+    st.session_state['manual_rating_history'],st.session_state['eval_df'], bool_rating_deleted = delete_last_manual_rating(
+        st.session_state['manual_rating_history'],st.session_state['eval_df'])
+    if_true_rerun(bool_rating_deleted)

pages/2_🤖Automated assessment.py

@@ -2,13 +2,15 @@ import streamlit as st
 import numpy as np
 from itertools import compress
 from PIL import Image
+from Dashboard_setup import sidebar_information
+sidebar_information() # Move this up to be displayed before the evaluation functions are loaded
 from Dashboard_automation_setup import fun_dict
 
 st.title('Automated Assessment')
 st.write('On this page you can use automated assessment algorithms to assess how good uploaded images match their respective prompts.')
 st.write(' ')
-st.sidebar.image('Graphics/IL_Logo.png')
 
+###### Setup of variables ############################
 try:
     # Create necessary variables
     prompt_dir = st.session_state['prompt_dir']
@@ -29,6 +31,7 @@ except KeyError:
     automated_eval_available = 0
 
 
+###### Rating loop ############################
 # If images for assessment available: create form to start assessment
 # Else: Note to upload images for assessment
 if automated_eval_available > 0:
@@ -56,14 +59,23 @@ if automated_eval_available > 0:
             # Create list for tasks which were selected for assessment
             selected_tasks = list(compress(task_list,task_list_selected))
 
-
             # Create dataset to loop over with assessment
             assessed_df = curr_eval_df.loc[
                     (curr_eval_df['automated_eval']==True)&
                     (curr_eval_df['Task'].isin(selected_tasks))]
             results_column = []
             
+            # Add counter for progress bars
+            num_automated_rows = len(assessed_df)
+            i_num_row = 0
+            i_progress_increase = 1/num_automated_rows
+            st.write('Progress of automatic evaluation:')
+            auto_assessment_progress = st.progress(0)
+
             for row in assessed_df.itertuples():
+                i_num_row +=1
+                auto_assessment_progress.progress(0+i_num_row*i_progress_increase)
+
                 # Apply task based classifier and safe in list
                 temp_image = Image.open(st.session_state['uploaded_img'][row.Picture_index])
                 temp_result = fun_dict[row.Task](
@@ -72,6 +84,6 @@ if automated_eval_available > 0:
 
             assessed_df['Score']=results_column
             st.session_state['auto_eval_df']=assessed_df[['File_name','Prompt_no','Picture_index','Task','Score']]
-            st.write('Completed assessment. Access results on the summary page.')
+            st.write('Assessment completed. You can access the results on the summary page. Running a new automated assessment will override past results.')
 else:
     st.write('Upload files on dashboard starting page to start automated assessment.')

pages/3_📊Assessment summary.py

@@ -4,22 +4,28 @@ import seaborn as sns
 import matplotlib.pyplot as plt
 from PIL import Image
 from pages.Functions.Dashboard_functions import pre_assessment_visualisation, multi_comparison_plotI, print_results_tabs
-side_image = Image.open('Graphics/IL_Logo.png')
-st.sidebar.image(side_image)
+from Dashboard_setup import sidebar_information, dashboard_version_code
+sidebar_information()
 
 
-@st.cache
-def convert_df_to_csv(df):
-  # IMPORTANT: Cache the conversion to prevent computation on every rerun
-  return df[['File_name','Prompt_no','Task','Score']].to_csv().encode('utf-8')
+#@st.cache
+#def convert_df_to_csv(df):
+# IMPORTANT: Cache the conversion to prevent computation on every rerun
+#  return df[['File_name','Prompt_no','Task','Score']].to_csv().encode('utf-8')
 
-assessment_result_frames = {}
 
+def df_to_csv_download(df, added_version_code='vNone'):
+  # IMPORTANT: Cache the conversion to prevent computation on every rerun
+  df['Dashboard_version']= added_version_code
+  return df[['File_name','Prompt_no','Task','Score','Dashboard_version']].to_csv().encode('utf-8')
 
+assessment_result_frames = {}
 st.title('Assessment Summary')
-st.header('Manual assessment')
 
 
+
+###### Manual assessment visualisation ############################
+st.header('Manual assessment')
 try:
   if sum(st.session_state['eval_df']['manual_eval_completed'])>0:
     # Display file uploader
@@ -29,7 +35,7 @@ try:
     manual_eval_df['Score'] = manual_eval_df['manual_eval_task_score'].map({'Yes':True, 'No':False})
     manual_results_df = manual_eval_df.loc[
       (manual_eval_df['manual_eval']==True)&
-      (manual_eval_df['manual_eval_completed']==True)]
+      ~(manual_eval_df['manual_eval_task_score'].isna())]
     manual_results_df['Model']='Manual assessment'
     assessment_result_frames['Manual assessment'] = manual_results_df
 
@@ -38,7 +44,7 @@ try:
 
     st.download_button(
       label="Download manual assessment data",
-      data=convert_df_to_csv(manual_results_df),
+      data=df_to_csv_download(manual_results_df, added_version_code=dashboard_version_code),
       file_name='manual_assessment.csv',
       mime='text/csv',
     )
@@ -47,6 +53,8 @@ try:
 except KeyError:
   pre_assessment_visualisation(type_str='manual')
 
+
+###### Automated assessment visualisation ############################
 st.write(' ')
 st.header('Automated assessment')
 try:
@@ -63,7 +71,7 @@ try:
 
   st.download_button(
     label="Download automated assessment data",
-    data=convert_df_to_csv(auto_eval_df),
+    data=df_to_csv_download(auto_eval_df, added_version_code=dashboard_version_code),
     file_name='automated_assessment.csv',
     mime='text/csv',
   )
@@ -71,6 +79,8 @@ except KeyError:
   pre_assessment_visualisation(type_str='automated')
 
 
+
+###### Gallery ############################
 try:
   # Start gallery
   st.header('Assessment gallery')
@@ -105,6 +115,7 @@ try:
   curr_Prompt = curr_prompt_dir[curr_prompt_dir['ID']==int(curr_Prompt_no)].Prompt
   curr_Picture_index = gallery_row_print.Picture_index.item()
   # Plot prompt and image
+  st.write('File name: '+gallery_row_print.File_name)
   st.write('Prompt: '+curr_Prompt.item())
   st.image(st.session_state['uploaded_img'][curr_Picture_index],width=350)
 

pages/Functions/Dashboard_functions.py

@@ -1,4 +1,8 @@
 # General functions and routines used in the dashboard
+'''
+- Functions below are ordered by page on which they are used
+- If possible, functions should not manipulate the session_state within them
+'''
 
 import streamlit as st
 import pandas as pd
@@ -9,6 +13,13 @@ from PIL import Image
 
 ##### Page-unspecific functions
 
+def if_true_rerun(bool_input):
+    '''
+    This function triggers a rerun of the page if the input == True
+    '''
+    if bool_input == True:
+        st.experimental_rerun()
+
 def assert_uploaded_frame(uploaded_df):
     # Set up variables checked for
     asserted_columns = {
@@ -19,7 +30,8 @@ def assert_uploaded_frame(uploaded_df):
     asserted_column_names = ['Prompt_no','Score','Task','File_name']
 
     # Check whether all needed column names are present
-    existing_column_names = [(x in uploaded_df.columns) for x in asserted_column_names]
+    df_columns_list = uploaded_df.columns.tolist()
+    existing_column_names = [(x in df_columns_list) for x in asserted_column_names]
     assert all(existing_column_names), "The uploaded dataframe is missing a column needed for import. Your table needs to contain the columns: 'Prompt_no', 'Score', 'Task', 'File_name' "
 
     # Check whether all needed columns have correct dtypes
@@ -35,74 +47,249 @@ def assert_multi_frame_upload(list_of_uploaded_dfs):
         assert_uploaded_frame(i_df)
 
 ##### Dashboard main page
-def prompt_to_csv(df):
+def prompt_to_csv(df, added_version_code='vNone'):
     df_download = df
-    df_download['Filename']='p'+df_download['ID'].astype('str')+'_1.png'
+    df_download['Filename']='p'+df_download['ID'].astype('str')+'_1_'+added_version_code+'.png'
     df_download = df[['Prompt','Filename']].drop_duplicates(subset='Filename')
     return df_download.to_csv().encode('utf-8')
 
+def prompt_df_for_download(prompt_dir):
+    '''
+    Function to create a subset of the prompt_dir via count based selection
+    '''
+    # Create local copy of variables
+    temp_prompt_dir = prompt_dir
+
+    # Create dict to hold counts of downloaded prompts
+    prompt_download_dict = {}
+    ## Count how many prompts are in database to allow for max value in selection
+    prompt_task_count = temp_prompt_dir.Task.value_counts(sort=False)
+    prompt_task_select = prompt_task_count.copy()
+
+    # Create numerical selector for every task in prompt directory, add count per task to dict
+    for i_task in prompt_task_select.index:
+        prompt_task_select[i_task] = st.number_input(
+            i_task,
+            value = prompt_task_count[i_task], 
+            max_value=prompt_task_count[i_task],
+            min_value=0,
+            step = 1)
+
+    # Create df with selected number of prompts per task
+    for i_task in prompt_task_select.index:
+        temp_df = temp_prompt_dir.loc[temp_prompt_dir['Task']==i_task][0:prompt_task_select[i_task]]
+        if len(temp_df)>0:
+            prompt_download_dict[i_task]=temp_df
+
+    # Concat all tasks to dataframe
+    prompt_download = pd.concat(prompt_download_dict.values())
+
+    # Add linked prompts, if the user chooses to
+    download_linked_prompts = st.checkbox('Download linked prompts', value=True)
+    if download_linked_prompts:
+
+        # Delete rows which do not have linked prompts to avoid type error
+        linked_prompts_info = prompt_download.dropna(subset='Linked_prompts')
+
+        # Add relevant linked prompts
+        linked_prompts_ids = linked_prompts_info.Linked_prompts.str.split(',').explode().unique().astype('int')
+        prompt_download = pd.concat(
+            [prompt_download,
+            temp_prompt_dir.loc[temp_prompt_dir['ID'].isin(linked_prompts_ids)]])
+
+        # Drop rows prompts which appear twice
+        prompt_download = prompt_download.drop_duplicates(subset='ID')
+
+    return prompt_download
+
 ##### Manual assessment
 
-def delete_last_manual_rating():
+def set_eval_df_rating_vals(eval_df, picture_index, manual_eval, manual_eval_completed, manual_eval_task_score):
+    '''
+    Function to set a block of key manual rating related variables of eval_df
+    '''
+    temp_eval_df = eval_df
+    temp_eval_df.loc[picture_index,'manual_eval']=manual_eval
+    temp_eval_df.loc[picture_index,'manual_eval_completed']=manual_eval_completed
+    temp_eval_df.loc[picture_index,'manual_eval_task_score']=manual_eval_task_score
+    return temp_eval_df 
+
+def radio_rating_index_translation(manual_rating_value):
+    if manual_rating_value == "No":
+        return 1
+    else:
+        return 0 
+
+
+def collect_linked_prompt_ratings(curr_linked_prompts, curr_eval_df, curr_prompt_dir):
+    '''
+    Create elements to collect ratings on linked prompts:
+    If there are linked prompts, create df with info
+    Else create emtpy df which will automatically skip the rating creation for these prompts
+    Here we do not test for (curr_eval_df['manual_eval']==True) as the curr_linked_prompts 
+    is already testing for valid prompt number and we want to ignore the exclusion for subprompts
+    '''
+    if type(curr_linked_prompts)==list:
+        curr_linked_rows = curr_eval_df.loc[
+            (curr_eval_df['manual_eval_completed']==False)&
+            (curr_eval_df['Prompt_no'].isin(curr_linked_prompts))]
+        curr_linked_rows = curr_linked_rows.groupby('Prompt_no').first()
+    else:
+        curr_linked_rows = pd.DataFrame()
+
+    # Create rating for subprompts if a df for subprompt info was created
+    for row in curr_linked_rows.itertuples():
+        # Preselected radio option
+        radio_preselect = radio_rating_index_translation(row.manual_eval_task_score)
+        # Prompt
+        st.write('Prompt: {0}'.format(
+            curr_prompt_dir.loc[curr_prompt_dir['ID']==int(row.Index)]['Prompt'].item()
+        ))
+        # Image
+        st.image(st.session_state['uploaded_img'][row.Picture_index],width=350)
+        # Rating
+        curr_linked_rows.loc[curr_linked_rows['Picture_index']==row.Picture_index,'manual_eval_task_score'] = st.radio(
+            "Does the image match the prompt?",('Yes', 'No'), horizontal=True, key=row.Picture_index, index=radio_preselect)
+        st.write(' ')
+        st.write(' ')
+
+    return curr_linked_rows
+
+
+def delete_last_manual_rating(session_history, eval_df):
     '''
     Routine to delete last manual rating and hence to return to it
     '''
-    if len(st.session_state['manual_rating_history'])>0:
+    # Create local copies of objects
+    temp_session_history = session_history
+    temp_eval_df = eval_df.copy()
+    temp_submit = False
 
+    if len(temp_session_history)>0:
         if st.button('Return to last rated image'):
             # The list contains sublists of images rated together, here we loop over these images to reset all of them
-            deleted_picture_index_list = st.session_state['manual_rating_history'].pop()
+            deleted_picture_index_list = temp_session_history.pop()
             for i_picind in deleted_picture_index_list:
-                st.session_state['eval_df'].loc[
+                temp_eval_df.loc[
                     i_picind,'manual_eval_completed']=False
-                st.session_state['eval_df'].loc[
-                    i_picind,'manual_eval_task_score']=np.nan  
-            st.experimental_rerun() 
+                #temp_eval_df.loc[
+                #    i_picind,'manual_eval_task_score']=np.nan  
+            
+            # Set submit boolean to true, to rerun the page
+            temp_submit = True
+    
+    return temp_session_history, temp_eval_df, temp_submit
+     
 
-def add_previous_manual_assessments():
+def add_previous_manual_assessments_upload_back(eval_df):
     '''
-    This is a routine to allow the user to upload prior manual ratings and override
-    current ratings. This way the user can restart a manual assessment.
+    Routine to upload a dataframe of previous (manual) assessment to add it to existing database.
+    The uploaded df is assessed, matching counts are printed and it returns the imported df for furthe processing.
     '''
-    # Create dict to translate uploaded score into str format used during manual assessment
-    Bool_str_dict = {True:'Yes',False:'No'}
-
-    st.subheader('Add previous assessments')
-    st.write('Upload results of previous assessment (as downloaded from summary page) to add these results and skip these images in your current manual assessment. Note that you can only add results for images which you have uploaded using the same file name.')
+    # Create necessary local variables
+    temp_eval_df = eval_df
 
-    uploaded_ratings = st.file_uploader('Select .csv for upload', accept_multiple_files=False)
-    if uploaded_ratings != None:
+    # Upload single dataframe, setting default to None for code type checking
+    temp_uploaded_ratings = None
+    temp_uploaded_ratings = st.file_uploader('Select .csv for upload', accept_multiple_files=False)
+    if temp_uploaded_ratings != None:
         try:
-            uploaded_ratings_df = pd.read_csv(uploaded_ratings)
+            # Import the uploaded csv as dataframe
+            uploaded_ratings_df = pd.read_csv(temp_uploaded_ratings)
             
             # Run standard assert pipeline
             assert_uploaded_frame(uploaded_ratings_df)
 
             # Show matching image count and instructions
-            overlapping_files_df =pd.merge(st.session_state['eval_df'],uploaded_ratings_df,on='File_name',how='inner')
+            overlapping_files_df = pd.merge(temp_eval_df,uploaded_ratings_df,on='File_name',how='inner')
             st.write('Number of matching file names found: '+ str(len(overlapping_files_df)))
             st.write('Click "Add results" button to add / override current ratings with uploaded ratings.')
+
+            return uploaded_ratings_df
         except UnicodeDecodeError:
             st.write('WARNING: The uploaded file has to be a .csv downloaded from the "Assessment summary" page.')
+    return temp_uploaded_ratings
 
 
-    submitted = st.button("Add results")
-    if submitted:
+def add_previous_manual_assessments_upload(eval_df, dashboard_version_code='vNone'):
+    '''
+    Routine to upload a dataframe of previous (manual) assessment to add it to existing database.
+    The uploaded df is assessed, matching counts are printed and it returns the imported df for furthe processing.
+    '''
+    # Create necessary local variables
+    temp_eval_df = eval_df
+
+    # Upload single dataframe, setting default to None for code type checking
+    temp_uploaded_ratings = None
+    temp_uploaded_ratings = st.file_uploader('Select .csv for upload', accept_multiple_files=False)
+    if temp_uploaded_ratings != None:
         try:
-            for row in uploaded_ratings_df.itertuples():
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval']=True
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_completed']=True
-                st.session_state['eval_df'].loc[
-                    st.session_state['eval_df']['File_name']==row.File_name,'manual_eval_task_score']=Bool_str_dict[row.Score]
+            # Import the uploaded csv as dataframe
+            uploaded_ratings_df = pd.read_csv(temp_uploaded_ratings)
+            
+            # Run standard assert pipeline
+            assert_uploaded_frame(uploaded_ratings_df)
+
+            # Check the uploaded df has a registered dashboard version
+            assert 'Dashboard_version' in uploaded_ratings_df.columns,"The uploaded dataframe needs to have a Dashboard_version column."
+            # Check for correct dashboard version in uploaded file
+            matching_dashboard_version = uploaded_ratings_df['Dashboard_version'] == dashboard_version_code
+            assert all(matching_dashboard_version),"The dashboard version of your uploaded results does not match the version of this dashboard."
+
+            # Show matching image count and instructions
+            overlapping_files_df = pd.merge(temp_eval_df,uploaded_ratings_df,on='File_name',how='inner')
+            st.write('Number of matching file names found: '+ str(len(overlapping_files_df)))
+            ## Show warning if some of the matching images already have a rating
+            if len(overlapping_files_df.manual_eval_task_score.dropna())>0:
+                st.write('WARNING: {0} of {1} matching files already have a saved rating. These will be overriden when you click "Add results".'.format(
+                    str(len(overlapping_files_df.manual_eval_task_score.dropna())),str(len(overlapping_files_df))))
+            st.write('Click "Add results" button to add uploaded ratings to current ratings.')
+            return uploaded_ratings_df
+        except UnicodeDecodeError:
+            st.write('WARNING: The uploaded file has to be a .csv downloaded from the "Assessment summary" page.')
+    return temp_uploaded_ratings
+
+def add_previous_manual_assessments_submit(eval_df, uploaded_ratings):
+    '''
+    If uploaded_ratings != None, this will create a button which when pressed will trigger
+    for the provided ratings to be added to eval_df
+    '''
+    # Create necessary local variables
+    temp_eval_df = eval_df
+    temp_submitted = False
+
+    # Create dict to translate uploaded score into str format used during manual assessment
+    bool_str_dict = {True:'Yes',False:'No'}
 
-            # Reset page after ratings were submitted
-            st.experimental_rerun()
-        except NameError:
-            st.write('You need to upload a .csv file before you can add results.')
+    # If a dataframe of uploaded ratings was provided: create a button which allows to add ratings to existing eval_df
+    if type(uploaded_ratings) == pd.DataFrame:
+        temp_submitted = st.button("Add results")
+        if temp_submitted:
+            for row in uploaded_ratings.itertuples():
+                temp_eval_df.loc[temp_eval_df['File_name']==row.File_name,'manual_eval']=True
+                temp_eval_df.loc[temp_eval_df['File_name']==row.File_name,'manual_eval_completed']=True
+                temp_eval_df.loc[temp_eval_df['File_name']==row.File_name,'manual_eval_task_score']=bool_str_dict[row.Score]
+    return temp_eval_df, temp_submitted
 
 
+def add_previous_manual_assessments(eval_df, dashboard_version_code):
+    '''
+    Full routine to allow the user to upload past ratings and add these to eval_df
+    '''
+    st.subheader('Add previous assessments')
+    st.write('Upload results of previous assessment (as downloaded from summary page) to add these results and skip these images in your current manual assessment. Note that you can only add results for images which you have uploaded using the same file name.')
+
+    # Create necessary local variables
+    temp_eval_df = eval_df
+
+    # Allow user to upload .csv with prior ratings
+    uploaded_ratings = add_previous_manual_assessments_upload(temp_eval_df, dashboard_version_code)
+
+    # Add rating to eval_df, if some were uploaded
+    temp_eval_df, temp_submitted = add_previous_manual_assessments_submit(temp_eval_df, uploaded_ratings)
+
+    return temp_eval_df, temp_submitted
+
 ##### Assessment summary
 
 def print_results_tabs(file_upload, results_df):
@@ -167,63 +354,3 @@ def multi_comparison_plotI(results_df = None, uploaded_df_list = []):
     plt.xlabel(' ')
     plt.ylim(0, 100)
     return fig,grouped_series
-
-
-
-
-############## Functions no longer used, to be deleted
-
-def plot_style_simple(results_df, return_table = False):
-    '''
-    Simple plot function for plotting just one dataframe of results
-    '''
-    eval_sum = results_df.groupby('Task')['Score'].sum()
-    eval_count = results_df.groupby('Task')['Score'].count()
-    eval_share = (eval_sum/eval_count)*100
-
-    if return_table:
-        return_series = results_df.groupby('Task')['Score'].sum()/results_df.groupby('Task')['Score'].count()*100
-        return_series = return_series.rename('Percentage correct')
-        return return_series
-
-    # Add small amount to make the bars on plot not disappear
-    eval_share = eval_share+1
-
-    fig = plt.figure(figsize=(12, 3))
-    sns.barplot(x=eval_share.index, y=eval_share.values, palette='GnBu')
-    plt.xticks(rotation=-65)
-    plt.ylabel('Percentage correct')
-    plt.xlabel(' ')
-    return fig
-
-def plot_style_combined(results_df, uploaded_df = None, return_table=False):
-    '''
-    Plot function which can plot to dataframe for comparison
-    '''
-    # Create joined dataframe of results and uploadd_df
-    uploaded_results_df = uploaded_df
-    manual_results_df['Model']='Current'
-    uploaded_results_df['Model']='Uploaded'
-    results_df = pd.concat([manual_results_df,uploaded_results_df])
-
-    # Create scores for plot
-    eval_sum = results_df.groupby(['Model','Task'])['Score'].sum()
-    eval_count = results_df.groupby(['Model','Task'])['Score'].count()
-    eval_share = (eval_sum/eval_count)*100
-    eval_share = eval_share.reset_index()
-
-    if return_table:
-        return_series = results_df.groupby(['Task','Model'])['Score'].sum()/results_df.groupby(['Task','Model'])['Score'].count()*100
-        return_series = return_series.rename('Percentage correct')
-        return return_series
-
-    # Add small amount to make the bars on plot not disappear
-    eval_share['Score'] = eval_share['Score']+1
-
-    # Create plot
-    fig = plt.figure(figsize=(12, 3))
-    sns.barplot(data=eval_share,x='Task',y='Score',hue='Model', palette='GnBu')
-    plt.xticks(rotation=-65)
-    plt.ylabel('Percentage correct')
-    plt.xlabel(' ')
-    return fig