Quality_Control.py

#!/usr/bin/env python
# coding: utf-8

import warnings
import os
import plotly as plt
import seaborn as sb
import plotly.express as px
import panel as pn
import holoviews as hv
import hvplot.pandas
import pandas as pd
import numpy as np
import json
import matplotlib.pyplot as plt
from bokeh.plotting import figure
from bokeh.io import push_notebook, show
from bokeh.io.export import export_png
from bokeh.resources import INLINE
from bokeh.embed import file_html
from bokeh.io import curdoc
from bokeh.models import Span, Label
from bokeh.models import ColumnDataSource, Button
from my_modules import *

#Silence FutureWarnings & UserWarnings
warnings.filterwarnings('ignore', category= FutureWarning)
warnings.filterwarnings('ignore', category= UserWarning)


'''get_ipython().run_line_magic('store', '-r base_dir')
get_ipython().run_line_magic('store', '-r set_path')
get_ipython().run_line_magic('store', '-r ls_samples')
get_ipython().run_line_magic('store', '-r selected_metadata_files')'''


'''# Retrieve the variables from the JSON file
with open('stored_variables.json', 'r') as file:
    stored_vars = json.load(file)
    
base_dir = stored_vars['base_dir']
set_path = stored_vars['set_path']
selected_metadata_files = stored_vars['selected_metadata_files']
ls_samples =  stored_vars['ls_samples']
print(f"Base Directory: {base_dir}")
print(f"Set Path: {set_path}")
print(f"Selected_metadata_files: {selected_metadata_files}")


print(base_dir)
print(set_path)
print(ls_samples)
print(selected_metadata_files)'''


    
base_dir = 'wetransfer_data-zip_2024-05-17_1431'
set_path = 'test'
selected_metadata_files = "['Slide_B_DD1s1.one_1.tif.csv', 'Slide_B_DD1s1.one_2.tif.csv']"
ls_samples = "['Ashlar_Exposure_Time.csv', 'new_data.csv', 'DD3S1.csv', 'DD3S2.csv', 'DD3S3.csv', 'TMA.csv']"

pn.extension()

update_button = pn.widgets.Button(name='CSV Files', button_type='primary')
def update_samples(event):
    with open('stored_variables.json', 'r') as file:
        stored_vars = json.load(file)
        ls_samples = stored_vars['ls_samples']
    print(ls_samples)
update_button.on_click(update_samples)

csv_files_button = pn.widgets.Button(icon="clipboard", button_type="primary")
indicator = pn.indicators.LoadingSpinner(value=False, size=25)

def handle_click(clicks):
    with open('stored_variables.json', 'r') as file:
        stored_vars = json.load(file)
        ls_samples = stored_vars['ls_samples']
    return f'CSV Files Selected: {ls_samples}'

pn.Row(
    csv_files_button,
    pn.bind(handle_click, csv_files_button.param.clicks),
)


# ## I.2. *DIRECTORIES

set_path = 'test'

# Set base directory

directorio_actual = os.getcwd()
print(directorio_actual)

##### MAC WORKSTATION #####
#base_dir = r'/Volumes/LaboLabrie/Projets/OC_TMA_Pejovic/Temp/Zoe/CyCIF_pipeline/'
###########################

##### WINDOWS WORKSTATION #####
#base_dir = r'C:\Users\LaboLabrie\gerz2701\cyCIF-pipeline\Set_B'
###############################
input_path = base_dir

##### LOCAL WORKSTATION #####
#base_dir = r'/Users/harshithakolipaka/Downloads/wetransfer_data-zip_2024-05-17_1431/'
base_dir = input_path
print(base_dir)
#############################

#set_name = 'Set_A'
#set_name = 'test'
set_name = set_path

project_name = set_name              # Project name
step_suffix = 'qc_eda'               # Curent part (here part I)
previous_step_suffix_long = ""       # Previous part (here empty)

# Initial input data directory
input_data_dir = os.path.join(base_dir, project_name + "_data")

# QC/EDA output directories
# global output
output_data_dir = os.path.join(base_dir, project_name + "_" + step_suffix)
# images subdirectory
output_images_dir = os.path.join(output_data_dir,"images")

# Data and Metadata directories
# global data
metadata_dir = os.path.join(base_dir, project_name + "_metadata")
# images subdirectory
metadata_images_dir = os.path.join(metadata_dir,"images")

# Create directories if they don't already exist
for d in [base_dir, input_data_dir, output_data_dir, output_images_dir, metadata_dir, metadata_images_dir]:
    if not os.path.exists(d):
        print("Creation of the" , d, "directory...")
        os.makedirs(d)
    else :
        print("The", d, "directory already exists !")

os.chdir(input_data_dir)
with open('stored_variables.json', 'r') as file:
        stored_vars = json.load(file)
        ls_samples = stored_vars['ls_samples']
        selected_metadata_files = stored_vars['selected_metadata_files']

directories = []
for i in [base_dir, input_data_dir, output_data_dir, output_images_dir, metadata_dir, metadata_images_dir]:
    directories.append(i)

directories

def print_directories(directories):
    
    label_path = []
    labels = [
        "base_dir",
        "input_data_dir",
        "output_data_dir",
        "output_images_dir",
        "metadata_dir",
        "metadata_images_dir"
    ]

    for label, path in zip(labels, directories):
        label_path.append(f"{label} : {path}")
        
    return label_path

print_directories


# Verify paths
print('base_dir :', base_dir)
print('input_data_dir :', input_data_dir)
print('output_data_dir :', output_data_dir)
print('output_images_dir :', output_images_dir)
print('metadata_dir :', metadata_dir)
print('metadata_images_dir :', metadata_images_dir)


# ## I.3. FILES

# Listing all the .csv files in the metadata/data directory
# Don't forget to move the csv files into the proj_data directory
# if the data dir is empty it's not going to work 
#ls_samples = [sample for sample in os.listdir(input_data_dir) if sample.endswith(".csv")]
print("The following CSV files were detected:\n\n",[sample for sample in ls_samples], "\n\nin", input_data_dir, "directory.")


# In[26]:


import os
import pandas as pd

def combine_and_save_metadata_files(metadata_dir, selected_metadata_files):   
    if len(selected_metadata_files) == []:
        if not file:
            warnings.warn("No Ashlar file uploaded. Please upload a valid file.", UserWarning)
            return
    
    elif len(selected_metadata_files) > 1:
        combined_metadata_df = pd.DataFrame()

        for file in selected_metadata_files:
            file_path = os.path.join(metadata_dir, file)
            df = pd.read_csv(file_path)
            combined_metadata_df = pd.concat([combined_metadata_df, df], ignore_index=True)

        combined_metadata_df.to_csv(os.path.join(metadata_dir, "combined_metadata.csv"), index=False)
        print(f"Combined metadata file saved as 'combined_metadata.csv' in {metadata_dir}")

        return combined_metadata_df
    
    else:
        if selected_metadata_files:
            single_file_path = os.path.join(metadata_dir, selected_metadata_files[0])
            single_file_df = pd.read_csv(single_file_path)
            print(f"Only one file selected: {selected_metadata_files[0]}")
            return single_file_df
        else:
            print("No metadata files selected.")
            return pd.DataFrame()


# In[27]:


print(combine_and_save_metadata_files(metadata_dir, selected_metadata_files))


# In[28]:


ls_samples


# In[29]:


df = pd.read_csv(os.path.join(input_data_dir, ls_samples[0]),index_col = 0, nrows = 1)
df.head(10)


# In[30]:


# First gather information on expected headers using first file in ls_samples
# Read in the first row of the file corresponding to the first sample (index = 0) in ls_samples
df = pd.read_csv(os.path.join(input_data_dir, ls_samples[0]) , index_col = 0, nrows = 1)


# Make sure the file was imported correctly
print("df :\n", df.head(), "\n")
print("df's columns :\n", df.columns, "\n")
print("df's index :\n", df.index, "\n")
print("df's index name :\n", df.index.name)


# In[31]:


df.head()


# In[32]:


# Verify that the ID column in input file became the index
# Verify that the index name column is "ID", if not, rename it
if df.index.name != "ID":
    print("Expected the first column in input file (index_col = 0) to be 'ID'. \n"
          "This column will be used to set the index names (cell number for each sample). \n"
          "It appears that the column '" + df.index.name + "' was actually the imported as the index column.")
    #df.index.name = 'ID'
    print("A new index name (first column) will be given ('ID') to replace the current one '" + df.index.name + "'\n")

# Apply the changes to the headers as specified with apply_header_changes() function (in my_modules.py)
# Apply the changes to the dataframe rows as specified with apply_df_changes() function (in my_modules.py)
#df = apply_header_changes(df)
print(df.index)
df.index = df.index.str.replace(r'@1$', '')
df = apply_df_changes(df)

# Set variable to hold default header values
expected_headers = df.columns.values
expected_header = True 
print(expected_header)

intial_dataframe = df
# Make sure the file is now formated correctly
print("\ndf :\n", df.head(), "\n")
print("df's columns :\n", df.columns, "\n")
print("df's index :\n", df.index, "\n")
print("df's index name :\n", df.index.name)


# In[33]:


df.head()


# In[34]:


df.head()


# In[35]:


print("Used " + ls_samples[0] + " to determine the expected and corrected headers for all files.\n")
print("These headers are: \n" + ", ".join([h for h in expected_headers]))

corrected_headers = True


# In[36]:


for sample in ls_samples:
    file_path = os.path.join(input_data_dir,sample)
    print(file_path)


# In[37]:


# Import all the others files
dfs = {}
###############################
# !! This may take a while !! #
###############################
errors = []

for sample in ls_samples:
    file_path = os.path.join(input_data_dir,sample)
    
    try:
        # Read the CSV file
        df = pd.read_csv(file_path, index_col=0)
        # Check if the DataFrame is empty, if so, don't continue trying to process df and remove it
        
        if not df.empty:
            # Manipulations necessary for concatenation
            df = apply_header_changes(df)
            df = apply_df_changes(df)
            # Reorder the columns to match the expected headers list
            #df = df.reindex(columns=expected_headers)
            print(df.head(1))
            print(sample, "file is processed !\n")
            #print(df) 
    
            # Compare df's header df against what is expected
            compare_headers(expected_headers, df.columns.values, sample)
            #print(df.columns.values)
            # Add a new colunm to identify the csv file (sample) where the df comes from
            df['Sample_ID'] = sample 
   
    except pd.errors.EmptyDataError:
        errors.append(f'\nEmpty data error in {sample} file. Removing from analysis...')
        print(f'\nEmpty data error in {sample} file. Removing from analysis...')
        ls_samples.remove(sample)      
    
    # Add df to dfs 
    dfs[sample] = df
    
print(dfs)


dfs.values()

# Merge dfs into one df
df = pd.concat(dfs.values(), ignore_index=False , sort = False)
del dfs
merge = True 
merged_dataframe = df
df.head()

# Set index to Sample_ID + cell number : 
# create a new custom index for df based on the sample names and integer cell numbers, and then remove the temporary columns 'level_0' and 'index' that were introduced during the operations

# Creates a copy of the DataFrame df and resets its index without creating a new column for the old index
# This essentially removes the old index column and replaces it with a default integer index
df = df.copy().reset_index(drop=True)

#print(df)

# Initializing an empty list index to store the new index labels for the DataFrame
index = []

for sample in ls_samples:
    # Extract a chunk of data from the original df where the 'Sample_ID' column matches the current sample name
    # This chunk is stored in the df_chunk df, which is a subset of the original data for that specific sample
    df_chunk = df.loc[df['Sample_ID'] == sample,:].copy()
    old_index = df_chunk.index
    # Reset the index of the df_chunk df, removing the old index and replacing it with a default integer index
    df_chunk = df_chunk.reset_index(drop=True)
    # A new index is created for the df_chunk df. It combines the sample name with 'Cell_' and the integer index values, converting them to strings
    # This new index will have labels like 'SampleName_Cell_0', 'SampleName_Cell_1', and so on.
    sample = sample.split('.')[0]
    df_chunk = df_chunk.set_index(f'{sample}_Cell_' + df_chunk.index.astype(str))
    # The index values of df_chunk are then added to the index list
    index = index + df_chunk.index.values.tolist()

# After processing all the samples in the loop, assign the index list as the new index of the original df.
df.index =  index
# Remove the 'level_0' and 'index' columns from df
df = df.loc[:,~df.columns.isin(['level_0','index'])]
assigned_new_index = True 
df.head()


# ### I.3.2. NOT_INTENSITIES

# not_intensities is the list of the columns unrelated to the markers fluorescence intensities
# Can include items that aren't in a given header.
#not_intensitiehttp://localhost:8888/lab/tree/Downloads/wetransfer_data-zip_2024-05-17_1431/1_qc_eda.ipynb
#I.3.2.-NOT_INTENSITIESs = ['Nuc_X', 'Nuc_X_Inv', 'Nuc_Y', 'Nuc_Y_Inv', 'Nucleus_Roundness', 'Nucleus_Size', 'Cell_Size', 
#                   'ROI_index', 'Sample_ID', 'replicate_ID', 'Cell_ID','cell_type', 'cell_subtype', 'cluster','ID', 
#                  'Cytoplasm_Size', 'immune_checkpoint', 'Unique_ROI_index', 'Patient', 'Primary_chem(1)_vs_surg(0)']
# not_intensities is the list of the columns unrelated to the markers fluorescence intensities
# Can include items that aren't in a given header.
#not_intensities = ['Nuc_X', 'Nuc_X_Inv', 'Nuc_Y', 'Nuc_Y_Inv', 'Nucleus_Roundness', 'Nucleus_Size', 'Cell_Size', 
#                   'ROI_index', 'Sample_ID', 'replicate_ID', 'Cell_ID','cell_type', 'cell_subtype', 'cluster','ID', 
#                   'Cytoplasm_Size', 'immune_checkpoint', 'Unique_ROI_index', 'Patient', 'Primary_chem(1)_vs_surg(0)']

# Get all column names
all_columns = df.columns.tolist()

# Create a list to store non-intensity column names
not_intensities = []
intensity_columns = []
# Iterate over each column name
for column in all_columns:
    # Check if the column name contains 'Intensity_Average'
    if 'Intensity_Average' not in column:
        print(not_intensities)
        not_intensities.append(column)
    else:
        intensity_columns.append(column)
    
    
# Create a new DataFrame with non-intensity columns
not_intensities_df = pd.DataFrame(not_intensities)
print("Non-intensity columns:")
print(not_intensities)

print("non-intensity DataFrame:")
not_intensities
#print(len(intensity_columns))


pd.DataFrame(not_intensities)

path_not_intensities = os.path.join(metadata_dir,"not_intensities.csv")

# If this file already exists, add only not_intensities items of the list not already present in file
if os.path.exists(path_not_intensities):
    print("'not_intensities.csv' already exists.")
    print("Reconciling file and Jupyter notebook lists.")
    file_not_intensities = open(path_not_intensities, "r")
    file_ni = file_not_intensities.read().splitlines()
    # Set difference to identify items not already in file
    to_add = set(not_intensities) - set(file_ni)
    # We want not_intensities to the a complete list
    not_intensities = list(set(file_ni) | set(not_intensities))
    file_not_intensities.close()
    file_not_intensities = open(path_not_intensities, "a")
    for item in to_add:
        file_not_intensities.write(item +"\n")
    file_not_intensities.close()

else:
    # The file does not yet exist
    print("Could not find " + path_not_intensities + ". Creating now.")
    file_not_intensities = open(path_not_intensities, "w")
    for item in not_intensities:
        file_not_intensities.write(item + "\n")
    file_not_intensities.close()


# In[46]:


not_intensities_df = pd.read_csv(path_not_intensities)
not_intensities_df


# In[47]:


# Columns we want to keep: not_intensities, and any intensity column that contains 'Intensity_Average' (drop any intensity marker column that is not a mean intensity)
to_keep = not_intensities + [x for x in df.columns.values[~df.columns.isin(not_intensities)] if 'Intensity_Average' in x]

to_keep


# In[48]:


print(len(to_keep) - 1)


# In[49]:


# However, our to_keep list contains items that might not be in our df headers!
# These items are from our not_intensities list. So let's ask for only those items from to_keep that are actually found in our df
# Retains only the columns from the to_keep list that are found in the df's headers (columns). 
# This ensures that we are only keeping the columns that exist in your df, avoiding any potential issues with non-existent column names. 
# The result is a df containing only the specified columns.
df = df[[x for x in to_keep if x in df.columns.values]]

df.head()


# In[50]:


import pandas as pd

# Assuming you have a DataFrame named 'df'
# df = pd.read_csv('your_file.csv')

# Get all column names
all_columns = df.columns.tolist()

# Create an empty list to store intensity markers
intensity_marker = []

# Iterate over each column name
for column in all_columns:
    # Check if the column name contains 'Intensity_Average'
    if 'Intensity_Average' in column:
        # Split the column name by underscore
        parts = column.split('_')
        
        # Extract the word before the first underscore
        marker = parts[0]
        
        # Add the marker to the intensity_marker list
        intensity_marker.append(marker)

# Remove duplicates from the intensity_marker list
intensity_marker = list(set(intensity_marker))

print("Intensity Markers:")
print(intensity_marker)

# Create a callback function to update the intensities array
def update_intensities(event):
    global intensities
    global intensities_df
    new_intensities = []
    selected_columns = []
    for marker, cell, cytoplasm, nucleus in zip(marker_options_df['Marker'], marker_options_df['Cell'], marker_options_df['Cytoplasm'], marker_options_df['Nucleus']):
        if cell:
            new_intensities.append(f"{marker}_Cell_Intensity_Average")
            selected_columns.append(f"{marker}_Cell_Intensity_Average")
        if cytoplasm:
            new_intensities.append(f"{marker}_Cytoplasm_Intensity_Average")
            selected_columns.append(f"{marker}_Cytoplasm_Intensity_Average")
        if nucleus:
            new_intensities.append(f"{marker}_Nucleus_Intensity_Average")
            selected_columns.append(f"{marker}_Nucleus_Intensity_Average")
    intensities = new_intensities
    if selected_columns:
        intensities_df = merged_dataframe[selected_columns]
    else:
        intensities_df = pd.DataFrame()
    print("Updated intensities DataFrame:")
    print(intensities_df)


# In[54]:


tabulator_formatters = {
    'bool': {'type': 'tickCross'}
}

# Create a DataFrame with the intensity markers and default values
marker_options_df = pd.DataFrame({
    'Marker': intensity_marker,
    'Cell': [False] * len(intensity_marker),
    'Cytoplasm': [False] * len(intensity_marker),
    'Nucleus': [False] * len(intensity_marker)
})

# Create the Tabulator widget and link the callback function
tabulator = pn.widgets.Tabulator(marker_options_df, formatters=tabulator_formatters, sizing_mode='stretch_width')
tabulator.param.watch(update_intensities,'value')

# Create a Panel layout with the Tabulator widget
marker_options_layout = pn.Column(tabulator, sizing_mode="stretch_width")

import panel as pn
import pandas as pd
import random
import asyncio

# Initialize the Panel extension with Tabulator
pn.extension('tabulator')

# Create a DataFrame with the intensity markers and default values
marker_options_df = pd.DataFrame({
    'Marker': intensity_marker,
    'Cell': [True] * len(intensity_marker),
    'Cytoplasm': [False] * len(intensity_marker),
    'Nucleus': [False] * len(intensity_marker)
})

# Define formatters for the Tabulator widget
tabulator_formatters = {
    'Cell': {'type': 'tickCross'},
    'Cytoplasm': {'type': 'tickCross'},
    'Nucleus': {'type': 'tickCross'}
}

# Create the Tabulator widget
tabulator = pn.widgets.Tabulator(marker_options_df, formatters=tabulator_formatters, sizing_mode='stretch_width')

# Create a DataFrame to store the initial intensities
new_data = [{'Description': f"{marker}_Cell_Intensity_Average"} for marker in intensity_marker if True]
new_data_df = pd.DataFrame(new_data)

# Create a widget to display the new data as a DataFrame
new_data_table = pn.widgets.Tabulator(new_data_df, name='New Data Table', sizing_mode='stretch_width')

# Create a button to start the update process
run_button = pn.widgets.Button(name="Save Selection", button_type='primary')

# Define the update_intensities function
def update_intensities():
    global new_data, new_data_df
    new_data = []
    for _, row in tabulator.value.iterrows():
        marker = row['Marker']
        if row['Cell']:
            new_data.append({'Description': f"{marker}_Cell_Intensity_Average"})
        if row['Cytoplasm']:
            new_data.append({'Description': f"{marker}_Cytoplasm_Intensity_Average"})
        if row['Nucleus']:
            new_data.append({'Description': f"{marker}_Nucleus_Intensity_Average"})
    new_data_df = pd.DataFrame(new_data)
    new_data_table.value = new_data_df

# Define the runner function
async def runner(event):
    update_intensities()

# Bind the runner function to the button
run_button.on_click(runner)

# Layout
updated_intensities = pn.Column(tabulator, run_button, new_data_table, sizing_mode="stretch_width")

pn.extension()
# Serve the layout
#updated_intensities.servable()


intensities_df = new_data_table
intensities_df

intensities_df = pn.pane.DataFrame(intensities_df)
intensities_df

print(intensities_df)
# ## I.4. QC CHECKS

def quality_check_results(check_shape, check_no_null,check_zero_intensities):
    results = [
        f"Check Index: {check_index}",
        f"Check Shape: {check_shape}",
        f"Check No Null: {check_no_null}",
        f"Check Zero Intensities: {check_zero_intensities}"
    ]
    return pn.Column(*[pn.Row(result) for result in results], sizing_mode="stretch_width")

print(ls_samples)

def check_index_format(index_str, ls_samples):
    """
    Checks if the given index string follows the specified format.

    Args:
        index_str (str): The index string to be checked.
        ls_samples (list): A list of valid sample names.

    Returns:
        bool: True if the index string follows the format, False otherwise.
    """
    # Split the index string into parts
    parts = index_str.split('_')

    # Check if there are exactly 3 parts
    if len(parts) != 3:
        print(len(parts))
        return False

    # Check if the first part is in ls_samples
    sample_name = parts[0]
    if f'{sample_name}.csv' not in ls_samples:
        print(sample_name)
        return False

    # Check if the second part is in ['cell', 'cytoplasm', 'nucleus']
    location = parts[1]
    valid_locations = ['Cell', 'Cytoplasm', 'Nucleus']
    if location not in valid_locations:
        print(location)
        return False

    # Check if the third part is a number
    try:
        index = int(parts[2])
    except ValueError:
        print(index)
        return False

    # If all checks pass, return True
    return True


# In[70]:


# Let's take a look at a few features to make sure our dataframe is as expected
df.index
def check_format_ofindex(index):
    for index in df.index:
        check_index = check_index_format(index, ls_samples) 
        if check_index is False:
            index_format = "Bad"
            return index_format
        
    index_format = "Good"   
    return index_format
print(check_format_ofindex(df.index))


# In[71]:


df.shape
check_index = df.index
check_shape = df.shape
print(check_shape)


# In[72]:


# Check for NaN entries (should not be any unless columns do not align)
# False means no NaN entries 
# True means NaN entries 
df.isnull().any().any()

check_no_null = df.isnull().any().any()


# In[73]:


# Check that all expected files were imported into final dataframe
if sorted(df.Sample_ID.unique()) == sorted(ls_samples):
    print("All expected filenames are present in big df Sample_ID column.")
    check_all_expected_files_present = "All expected filenames are present in big df Sample_ID column."
else:
    compare_headers(['no samples'], df.Sample_ID.unique(), "big df Sample_ID column")
    check_all_expected_files_present = compare_headers(['no samples'], df.Sample_ID.unique(), "big df Sample_ID column")

print(df.Sample_ID)


# In[74]:


# Delete rows that have 0 value mean intensities for intensity columns
print("df.shape before removing 0 mean values: ", df.shape)

# We use the apply method on df to calculate the mean intensity for each row. It's done this by applying a lambda function to each row. 
# The lambda function excludes the columns listed in the not_intensities list (which are not to be considered for mean intensity calculations) 
# and calculates the mean of the remaining values in each row.
###############################
# !! This may take a while !! #
###############################
# Calculate mean intensity excluding 'not_intensities' columns
mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)

# Check if there are any 0 mean intensity values
if (mean_intensity == 0).any():
    df = df.loc[mean_intensity > 0, :]
    print("Shape after removing 0 mean values: ", df.shape) 
    check_zero_intensities = f'df.shape after removing 0 mean values: {df.shape}'
else:
    print("No zero intensity values.")
    check_zero_intensities = " No zero intensity values found in the DataFrame."



# Get quantiles (5th, 50th, 95th)
# List of nucleus size percentiles to extract 
#qs = [0.05,0.50,0.95] 



#df["Nucleus_Size"].quantile(q=qs)


quality_control_df = df
quality_control_df.head()

# Function to perform quality checks
def perform_quality_checks(df, ls_samples, not_intensities):
    results = {}
    errors = []
    # Check index
    results['index'] = df.index
    
    # Check shape
    results['shape'] = df.shape
    
    # Check for NaN entries
    results['nan_entries'] = df.isnull().any().any()

    # Remove rows with 0 mean intensity values
    mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)
    if (mean_intensity == 0).any():
        df = df.loc[mean_intensity > 0, :]
        results['zero_intensity_removal'] = f"Zero intensity entires are found and removed. Shape after removing: {df.shape}"
    else:
        results['zero_intensity_removal'] = "No zero intensity values found in the DataFrame."
    
    return results

# Example usage of the function
quality_check_results = perform_quality_checks(df, ls_samples, not_intensities)

# Print results
for key, value in quality_check_results.items():
    print(f"{key}: {value}")


# In[80]:


import panel as pn
import pandas as pd

def quality_check(file, not_intensities):
    # Load the output file
    df = file

    # Check Index
    check_index = check_format_ofindex(df.index)

    # Check Shape
    check_shape = df.shape

    # Check for NaN entries
    check_no_null = df.isnull().any().any()

    mean_intensity = df.loc[:, ~df.columns.isin(not_intensities)].mean(axis=1)
    if (mean_intensity == 0).any():
        df = df.loc[mean_intensity > 0, :]
        print("df.shape after removing 0 mean values: ", df.shape)
        check_zero_intensities = f'df.shape after removing 0 mean values: {df.shape}'
    else:
        print("No zero intensity values found in the DataFrame.")
        check_zero_intensities = "No zero intensities."

    # Create a quality check results table
    quality_check_results_table = pd.DataFrame({
        'Check': ['Index', 'Shape', 'Check for NaN Entries', 'Check for Zero Intensities'],
        'Result': [str(check_index), str(check_shape), str(check_no_null), check_zero_intensities]
    })

    # Create a quality check results component
    quality_check_results_component = pn.Card(
        pn.pane.DataFrame(quality_check_results_table),
        title="Quality Control Results",
        header_background="#2196f3",
        header_color="white",
    )

    return quality_check_results_component

quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)


# Function to calculate quantile values
def calculate_quantiles(quantile):
    quantile_value_intensity = df["AF555_Cell_Intensity_Average"].quantile(q=[quantile, 0.50, 1 - quantile])    
    return quantile_value_intensity

# Function to create the Panel app
def create_app(quantile = quantile_slider.param.value):
    quantiles = calculate_quantiles(quantile)
    output = pd.DataFrame(quantiles)
    
    # Create a Markdown widget to display the output
    output_widget = pn.pane.DataFrame(output)

    return output_widget


# Bind the create_app function to the quantile slider
quantile_output_app = pn.bind(create_app, quantile_slider.param.value)
#pn.Column(quantile_slider,quantile_output_app).servable()

# Function to create the line graph plot using Bokeh
def create_line_graph2(quantile):
    # Calculate histogram
    hist, edges = np.histogram(df['Nucleus_Size'], bins=30)
    
    # Calculate the midpoints of bins for plotting
    midpoints = (edges[:-1] + edges[1:]) / 2

    # Calculate quantiles
    qs = [quantile, 0.50, 1.00 - quantile]
    quantiles = df['Nucleus_Size'].quantile(q=qs).values

    # Create Bokeh line graph plot
    p = figure(title='Frequency vs. Nucleus_Size', 
               x_axis_label='Nucleus_Size', 
               y_axis_label='Frequency',
               width=800, height=400)
    
    # Plotting histogram
    p.quad(top=hist, bottom=0, left=edges[:-1], right=edges[1:],
           fill_color='skyblue', line_color='black', alpha=0.6)
    
    # Plotting line graph
    p.line(midpoints, hist, line_width=2, color='blue', alpha=0.7)
    
    # Add quantile lines
    for q in quantiles:
        span = Span(location=q, dimension='height', line_color='red', line_dash='dashed', line_width=2)
        p.add_layout(span)
        p.add_layout(Label(x=q, y=max(hist), text=f'{q:.1f}', text_color='red'))
    
    return p

# Bind the create_line_graph function to the quantile slider
nucleus_size_line_graph_with_histogram = pn.bind(create_line_graph2, quantile=quantile_slider.param.value)

# Layout the components in a Panel app
#nucleus_size_line_graph_with_histogram = pn.Column(create_line_graph2(quantile = quantile_slider.param.value))
#nucleus_size_line_graph_with_histogram.servable()
# Layout the components in a Panel app
plot1 = pn.Column(quantile_slider, pn.pane.Bokeh(nucleus_size_line_graph_with_histogram))
#plot1.servable()

#Removing cells based on nucleus size

quantile = quantile_slider.value
qs = [quantile, 0.50, 1.00 - quantile]
quantiles = df['Nucleus_Size'].quantile(q=qs).values
threshold = quantiles[2]


# In[89]:


print(threshold)


# In[90]:



import panel as pn
import pandas as pd
import numpy as np
from bokeh.plotting import figure
from bokeh.models import Span, Label
# Define the quantile slider
#quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)

# Function to update the threshold and display number of cells removed
def update_threshold_and_display(quantile):
    qs = [quantile, 0.50, 1.00 - quantile]
    quantiles = df['Nucleus_Size'].quantile(q=qs).values
    threshold = quantiles[2]
    
    # Filter the DataFrame based on the new threshold
    df_filtered = df.loc[(df['Nucleus_Size'] > 42) & (df['Nucleus_Size'] < threshold)]
    
    # Calculate the number of cells removed
    cells_before_filter = df.shape[0]
    cells_after_filter = df_filtered.shape[0]
    cells_removed = cells_before_filter - cells_after_filter
    
    # Display the results
    results = pn.Column(
        f"Number of cells before filtering: {cells_before_filter}",
        f"Number of cells after filtering on nucleus size: {cells_after_filter}",
        f"Number of cells removed: {cells_removed}"
    )
    
    return results

# Bind the update function to the quantile slider
results_display = pn.bind(update_threshold_and_display, quantile_slider)

# Layout the components in a Panel app
layout2 = results_display


# In[91]:


print("Number of cells before filtering :", df.shape[0])
cells_before_filter = f"Number of cells before filtering :{df.shape[0]}"
# Delete small cells and objects w/high AF555 Signal (RBCs) 
# We usually use the 95th percentile calculated during QC_EDA
df = df.loc[(df['Nucleus_Size'] > 42 )]
df = df.loc[(df['Nucleus_Size'] < threshold)]
cells_after_filter_nucleus_shape = df.shape[0]
print("Number of cells after filtering on nucleus size:", df.shape[0])

df = df.loc[(df['AF555_Cell_Intensity_Average'] < 2000)]
print("Number of cells after filtering on AF555A ___ intensity:", df.shape[0])
cells_after_filter_intensity_shape = df.shape[0]
cells_after_filter_nucleus = f"Number of cells after filtering on nucleus size: {cells_after_filter_nucleus_shape}"
cells_after_filter_intensity = f"Number of cells after filtering on AF555A ___ intensity: {cells_after_filter_intensity_shape}"

num_of_cell_removal_intensity = cells_after_filter_intensity

print(num_of_cell_removal_intensity )

num_of_cell_removal = pn.Column(cells_before_filter, cells_after_filter_nucleus)


# Assuming you have a DataFrame 'df' with the intensity columns
intensities = df.filter(like='Intensity').columns.tolist()

# Create a ColumnDataSource from the DataFrame
source = ColumnDataSource(df)

# Function to calculate quantile values
def calculate_quantiles(column, quantile):
    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile]).values
    return quantiles

# Create the dropdown menu
column_dropdown = pn.widgets.Select(name='Select Column', options=intensities)

quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)


# Function to create the Bokeh plot
def create_intensity_plot(column, quantile):
    quantiles = calculate_quantiles(column, quantile)
    hist, edges = np.histogram(df[column], bins = 30)
    # Calculate the midpoints of bins for plotting
    midpoints = (edges[:-1] + edges[1:]) / 2
    
    # Create Bokeh plot
    p = figure(title=f'Distribution of {column} with Quantiles', 
               x_axis_label=f'{column} Values', 
               y_axis_label='Frequency',
               width=800, height=400)
    
    
    p.quad(top=hist, bottom=0, left=edges[:-1], right= edges[1:], 
           fill_color='skyblue', line_color='black', alpha=0.7)

    # Plotting line graph
    p.line(midpoints, hist, line_width=2, color='blue', alpha=0.7)
    
    # Add quantile lines
    for q in quantiles:
        span = Span(location=q, dimension='height', line_color='red', line_dash='dashed', line_width=2)
        p.add_layout(span)
        p.add_layout(Label(x=q, y=max(hist), text=f'{q:.1f}', text_color='red'))
    
    return p


# Bind the create_plot function to the quantile slider, column dropdown, and button click
marker_intensity_with_histogram = pn.bind(create_intensity_plot,column_dropdown.param.value, quantile_slider.param.value, watch=True)

# Create the button
generate_plot_button = Button(label='Generate Plot', button_type='primary')

def update_plot(column, quantile):
    plot = create_intensity_plot(column, quantile)
    plot.renderers[0].data_source = source  # Update the data source for the renderer
    return plot

#Display the dropdown menu, quantile slider, button, and plot
#plot = update_plot(column_dropdown.param.value, quantile_slider.param.value)

def generate_plot(event):
    updated_plot = update_plot(column_dropdown.param.value, quantile_slider.param.value)
    #pn.Column(pn.Row(column_dropdown, generate_plot_button), quantile_slider, updated_plot).servable()

generate_plot_button.on_click(generate_plot)
selected_marker_plot = pn.Column(pn.Row(pn.Column(column_dropdown, marker_intensity_with_histogram )))
#pn.Column(pn.Row(pn.Column(column_dropdown, marker_intensity_with_histogram ), generate_plot_button)).servable()

import panel as pn
import numpy as np
import pandas as pd
from bokeh.plotting import figure
from bokeh.models import ColumnDataSource, Button, Span, Label

# Assuming you have a DataFrame 'df' with the intensity columns
intensities = df.filter(like='Intensity').columns.tolist()

# Create a ColumnDataSource from the DataFrame
source = ColumnDataSource(df)

# Function to calculate quantile values
def calculate_quantiles(column, quantile):
    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
    return quantiles


# In[105]:


quantile_slider = pn.widgets.FloatSlider(name='Quantile', start=0.01, end=0.99, step=0.01, value=0.05)


# Bind the create_line_graph function to the quantile slider
#nucleus_size_line_graph = pn.bind(create_line_graph, quantile=quantile_slider.param.value)

# Layout the components in a Panel app
#nucleus_size_graph = pn.Column(nucleus_size_line_graph)


# In[106]:


#df["CKs_Cytoplasm_Intensity_Average"].quantile(q=qs)


# In[107]:


len(intensities)
if 'CKs_Cytoplasm_Intensity_Average' in intensities:
    print(1)


# In[108]:


df


# In[109]:


def calculate_cytoplasm_quantiles(column, quantile):
    # Print the columns of the DataFrame
    print("DataFrame columns:", df.columns)
    
    # Check if the column exists in the DataFrame
    if column not in df.columns:
        raise KeyError(f"Column '{column}' does not exist in the DataFrame.")
    
    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
    return quantiles

def create_cytoplasm_intensity_df(column, quantile):
    quantiles = calculate_cytoplasm_quantiles(column, quantile)
    output = pd.DataFrame(quantiles)
    return pn.pane.DataFrame(output)

# Bind the create_app function to the quantile slider
cytoplasm_quantile_output_app = pn.bind(create_cytoplasm_intensity_df, column='CKs_Cytoplasm_Intensity_Average', quantile=quantile_slider.param.value)

pn.Column(quantile_slider, cytoplasm_quantile_output_app)


# In[110]:


def calculate_cytoplasm_quantiles(column, quantile):
    quantiles = df[column].quantile(q=[quantile, 0.50, 1 - quantile])
    return quantiles

def create_cytoplasm_intensity_df(column, quantile):
    quantiles = calculate_cytoplasm_quantiles(column, quantile)
    output = pd.DataFrame(quantiles)
    # Create a Dataframe widget to display the output
    output_widget = pn.pane.DataFrame(output)
    return output_widget


# Bind the create_app function to the quantile slider
cytoplasm_quantile_output_app = pn.bind(create_cytoplasm_intensity_df, column='CKs_Cytoplasm_Intensity_Average', quantile = quantile_slider.param.value)
pn.Column(quantile_slider,cytoplasm_quantile_output_app)


# ## I.5. COLUMNS OF INTERESTS

# In[111]:


# Remove columns containing "DAPI"
df = df[[x for x in df.columns.values if 'DAPI' not in x]]

print("Columns are now...")
print([c for c in df.columns.values])


# In[112]:


# Create lists of full names and shortened names to use in plotting
full_to_short_names, short_to_full_names =  \
    shorten_feature_names(df.columns.values[~df.columns.isin(not_intensities)])

short_to_full_names


# In[113]:


# Save this data to a metadata file
filename = os.path.join(metadata_dir, "full_to_short_column_names.csv")
fh = open(filename, "w")
fh.write("full_name,short_name\n")
for k,v in full_to_short_names.items():
    fh.write(k + "," + v + "\n")
    
fh.close()
print("The full_to_short_column_names.csv file was created !")


# In[114]:


# Save this data to a metadata file
filename = os.path.join(metadata_dir, "short_to_full_column_names.csv")
fh = open(filename, "w")
fh.write("short_name,full_name\n")
for k,v in short_to_full_names.items():
    fh.write(k + "," + v + "\n")
    
fh.close()
print("The short_to_full_column_names.csv file was created !")


# ## I.6. EXPOSURE TIME

# In[115]:


#import the ashlar analysis file 
file_path = os.path.join(metadata_dir, 'combined_metadata.csv')
ashlar_analysis = pd.read_csv(file_path)
ashlar_analysis 


# In[116]:


# Extracting and renaming columns
new_df = ashlar_analysis[['Name', 'Cycle', 'ChannelIndex', 'ExposureTime']].copy()
new_df.rename(columns={
    'Name': 'Target',
    'Cycle': 'Round',
    'ChannelIndex': 'Channel'
}, inplace=True)

# Applying suffixes to the columns
new_df['Round'] = 'R' + new_df['Round'].astype(str)
new_df['Channel'] = 'c' + new_df['Channel'].astype(str) 

# Save to CSV
new_df.to_csv('Ashlar_Exposure_Time.csv', index=False)

# Print the new dataframe
print(new_df)


# In[117]:


# Here, we want to end up with a data structure that incorporates metadata on each intensity marker column used in our big dataframe in an easy-to-use format. 
# This is going to include the full name of the intensity marker columns in the big data frame, 
# the corresponding round and channel, 
# the target protein (e.g., CD45), 
# and the segmentation localization information (cell, cytoplasm, nucleus)

# We can use this data structure to assign unique colors to all channels and rounds, for example, for use in later visualizations
# Exposure_time file from ASHLAR analysis
filename = "Exposure_Time.csv"
filename = os.path.join(metadata_dir, filename)
exp_df = pd.read_csv(filename)

print(exp_df)

# Verify file imported correctly
# File length
print("df's shape: ", exp_df.shape)
# Headers
expected_headers =['Round','Target','Exp','Channel']
compare_headers(expected_headers, exp_df.columns.values, "Imported metadata file")

# Missingness
if exp_df.isnull().any().any():
    print("\nexp_df has null value(s) in row(s):")
    print(exp_df[exp_df.isna().any(axis=1)])
else:
    print("\nNo null values detected.")


# In[118]:


if len(exp_df['Target']) > len(exp_df['Target'].unique()):
    print("One or more non-unique Target values in exp_df. Currently not supported.")
exp_df = exp_df.drop_duplicates(subset = 'Target').reindex()


# In[119]:


# sort exp_df by the values in the 'Target' column in ascending order and then retrieve the first few rows of the sorted df
exp_df.sort_values(by = ['Target']).head()


# In[120]:


# Create lowercase version of target
exp_df['target_lower'] = exp_df['Target'].str.lower()
exp_df.head()


# In[121]:


# Create df that contains marker intensity columns in our df that aren't in not_intensities
intensities = pd.DataFrame({'full_column':df.columns.values[~df.columns.isin(not_intensities)]})

intensities


# In[122]:


# Extract the marker information from the `full_column`, which corresponds to full column in big dataframe
# Use regular expressions (regex) to isolate the part of the field that begins (^) with an alphanumeric value (W), and ends with an underscore (_)
# '$' is end of line
intensities['marker'] = intensities['full_column'].str.extract(r'([^\W_]+)')
# convert to lowercase
intensities['marker_lower'] = intensities['marker'].str.lower()

intensities


# In[123]:


# Subset the intensities df to exclude any column pertaining to DAPI
intensities = intensities.loc[intensities['marker_lower'] != 'dapi']

intensities.head()


# In[124]:


# Merge the intensities andexp_df together to create metadata
metadata = pd.merge(exp_df, intensities, how = 'left', left_on = 'target_lower',right_on = 'marker_lower')
metadata = metadata.drop(columns = ['marker_lower'])
metadata = metadata.dropna()

# Target is the capitalization from the Exposure_Time.csv
# target_lower is Target in small caps
# marker is the extracted first component of the full column in segmentation data, with corresponding capitalization
metadata


# In[125]:


# Add a column to signify marker target localisation.
# Use a lambda to determine segmented location of intensity marker column and update metadata accordingly
# Using the add_metadata_location() function in my_modules.py
metadata['localisation'] = metadata.apply(
    lambda row: add_metadata_location(row), axis = 1)


# In[126]:


mlid = metadata


# In[127]:


# Save this data structure to the metadata folder
# don't want to add color in because that's better off treating color the same for round, channel, and sample
filename = "marker_intensity_metadata.csv"
filename = os.path.join(metadata_dir, filename)
metadata.to_csv(filename, index = False)
print("The marker_intensity_metadata.csv file was created !")



# ## I.7. COLORS WORKFLOW

# ### I.7.1. CHANNELS COLORS


# we want colors that are categorical, since Channel is a non-ordered category (yes, they are numbered, but arbitrarily). 
# A categorical color palette will have dissimilar colors.
# Get those unique colors
if len(metadata.Channel.unique()) > 10:
    print("WARNING: There are more unique channel values than \
    there are colors to choose from. Select different palette, e.g., \
    continuous palette 'husl'.")
channel_color_values = sb.color_palette("bright",n_colors = len(metadata.Channel.unique()))
# chose 'colorblind' because it is categorical and we're unlikely to have > 10

# You can customize the colors for each channel here
custom_colors = {
    'c2': 'lightgreen',
    'c3': 'tomato',
    'c4': 'pink',
    'c5': 'turquoise'
}

custom_colors_values = sb.palplot(sb.color_palette([custom_colors.get(ch, 'blue') for ch in metadata.Channel.unique()]))

# Display those unique customs colors
print("Unique channels are:", metadata.Channel.unique())
sb.palplot(sb.color_palette(channel_color_values))


# In[131]:


# Function to create a palette plot with custom colors
def create_palette_plot():
    # Get unique channels
    unique_channels = metadata.Channel.unique()
    
    # Define custom colors for each channel
    custom_colors = {
        'c2': 'lightgreen',
        'c3': 'tomato',
        'c4': 'pink',
        'c5': 'turquoise'
    }
    
    # Get custom colors for each channel
    colors = [custom_colors.get(ch, 'blue') for ch in unique_channels]
    
    # Create a palette plot (palplot)
    palette_plot = sb.palplot(sb.color_palette(colors))
    channel_color_values = sb.color_palette("bright",n_colors = len(metadata.Channel.unique()))
    channel_color_values = sb.palplot(channel_color_values)
    return palette_plot, channel_color_values


# Create the palette plot directly
palette_plot = create_palette_plot()

# Define the Panel app layout
app_palette_plot = pn.Column(
    pn.pane.Markdown("### Custom Color Palette"),
    palette_plot, 
)

# Function to create a palette plot with custom colors
def create_palette_plot(custom_colors):
    # Get unique channels
    unique_channels = metadata.Channel.unique()
    
    # Get custom colors for each channel
    colors = [custom_colors.get(ch, 'blue') for ch in unique_channels]
    
    # Create a palette plot (palplot)
    palette_plot = sb.palplot(sb.color_palette(colors))
    
    return palette_plot

# Define custom colors for each channel
custom_colors = {
    'c2': 'lightgreen',
    'c3': 'tomato',
    'c4': 'pink',
    'c5': 'turquoise'
}

# Display those unique customs colo
print("Unique channels are:", metadata.Channel.unique())
# Function to bind create_palette_plot
app_palette_plot = create_palette_plot(custom_colors)


#app_palette_plot.servable()


# In[133]:


# Store in a dictionary
channel_color_dict = dict(zip(metadata.Channel.unique(), channel_color_values))
channel_color_dict
for k,v in channel_color_dict.items():
    channel_color_dict[k] = np.float64(v)

channel_color_dict


# In[134]:


color_df_channel = color_dict_to_df(channel_color_dict, "Channel")

# Save to file in metadatadirectory
filename = "channel_color_data.csv"
filename = os.path.join(metadata_dir, filename)
color_df_channel.to_csv(filename, index = False)

color_df_channel


# In[135]:


# Legend of channel info only
g  = plt.figure(figsize = (1,1)).add_subplot(111)
g.axis('off')
handles = []
for item in channel_color_dict.keys():
        h = g.bar(0,0, color = channel_color_dict[item],
                  label = item, linewidth =0)
        handles.append(h)
first_legend = plt.legend(handles=handles, loc='upper right', title = 'Channel'),
                            # box_to_anchor=(10,10), 
                             #       bbox_transform=plt.gcf().transFigure)

filename = "Channel_legend.png"
filename = os.path.join(metadata_images_dir, filename)
plt.savefig(filename, bbox_inches = 'tight')

# ### I.7.2. ROUNDS COLORS


# we want colors that are sequential, since Round is an ordered category. 
# We can still generate colors that are easy to distinguish. Also, many of the categorical palettes cap at at about 10 or so unique colors, and repeat from there. 
# We do not want any repeats!
round_color_values = sb.cubehelix_palette(
    len(metadata.Round.unique()), start=1, rot= -0.75, dark=0.19, light=.85, reverse=True)
# round_color_values = sb.color_palette("cubehelix",n_colors = len(metadata.Round.unique()))
# chose 'cubehelix' because it is sequential, and round is a continuous process
# each color value is a tuple of three values: (R, G, B)
print(metadata.Round.unique())

sb.palplot(sb.color_palette(round_color_values))

## TO-DO: write what these parameters mean


# In[137]:


# Store in a dictionary
round_color_dict = dict(zip(metadata.Round.unique(), round_color_values))

for k,v in round_color_dict.items():
    round_color_dict[k] = np.float64(v)

round_color_dict


# In[138]:


color_df_round = color_dict_to_df(round_color_dict, "Round")

# Save to file in metadatadirectory
filename = "round_color_data.csv"
filename = os.path.join(metadata_dir, filename)
color_df_round.to_csv(filename, index = False)

color_df_round

# Legend of round info only

round_legend  = plt.figure(figsize = (1,1)).add_subplot(111)
round_legend.axis('off')
handles = []
for item in round_color_dict.keys():
        h = round_legend.bar(0,0, color = round_color_dict[item],
                  label = item, linewidth =0)
        handles.append(h)
first_legend = plt.legend(handles=handles, loc='upper right', title = 'Round'),
                            # bbox_to_anchor=(10,10), 
                             #       bbox_transform=plt.gcf().transFigure)

filename = "Round_legend.png"
filename = os.path.join(metadata_images_dir, filename)
plt.savefig(filename, bbox_inches = 'tight')


# ### I.7.3. SAMPLES COLORS

# In[140]:


# we want colors that are neither sequential nor categorical. 
# Categorical would be ideal if we could generate an arbitrary number of colors, but I do not think that we can. 
# Hense, we will choose `n` colors from a continuous palette. First we will generate the right number of colors. Later, we will assign TMA samples to gray.

# Get those unique colors
color_values = sb.color_palette("husl",n_colors = len(ls_samples))#'HLS'
# each color value is a tuple of three values: (R, G, B)

# Display those unique colors
sb.palplot(sb.color_palette(color_values))


# In[141]:


TMA_samples = [s for s in df.Sample_ID.unique() if 'TMA' in s]
TMA_color_values = sb.color_palette(n_colors = len(TMA_samples),palette = "gray")
sb.palplot(sb.color_palette(TMA_color_values))


# In[142]:


# Store in a dictionary
color_dict = dict()
color_dict = dict(zip(df.Sample_ID.unique(), color_values))

# Replace all TMA samples' colors with gray
i = 0
for key in color_dict.keys():
    if 'TMA' in key:
        color_dict[key] = TMA_color_values[i]
        i +=1

color_dict

color_df_sample = color_dict_to_df(color_dict, "Sample_ID")

# Save to file in metadatadirectory
filename = "sample_color_data.csv"
filename = os.path.join(metadata_dir, filename)
color_df_sample.to_csv(filename, index = False)

color_df_sample


# Legend of sample info only
g  = plt.figure(figsize = (1,1)).add_subplot(111)
g.axis('off')
handles = []
for item in color_dict.keys():
        h = g.bar(0,0, color = color_dict[item],
                  label = item, linewidth =0)
        handles.append(h)
first_legend = plt.legend(handles=handles, loc='upper right', title = 'Sample')

filename = "Sample_legend.png"
filename = os.path.join(metadata_images_dir, filename)
plt.savefig(filename, bbox_inches = 'tight')


# ### I.7.4. CLUSTERS COLORS

'''if 'cluster' in df.columns:
    cluster_color_values = sb.color_palette("hls",n_colors = len(df.cluster.unique()))

    #print(sorted(test_df.cluster.unique()))
    # Display those unique colors
    sb.palplot(sb.color_palette(cluster_color_values))
    
    cluster_color_dict = dict(zip(sorted(test_df.cluster.unique()), cluster_color_values))
    print(cluster_color_dict)
    
    # Create dataframe
    cluster_color_df = color_dict_to_df(cluster_color_dict, "cluster")
    cluster_color_df.head()

    # Save to file in metadatadirectory
    filename = "cluster_color_data.csv"
    filename = os.path.join(metadata_dir, filename)
    cluster_color_df.to_csv(filename, index = False) 



# Legend of cluster info only

if 'cluster' in df.columns:
    g  = plt.figure(figsize = (1,1)).add_subplot(111)
    g.axis('off')
    handles = []
    for item in sorted(cluster_color_dict.keys()):
            h = g.bar(0,0, color = cluster_color_dict[item],
                      label = item, linewidth =0)
            handles.append(h)
    first_legend = plt.legend(handles=handles, loc='upper right', title = 'Cluster'),


    filename = "Clustertype_legend.png"
    filename = os.path.join(metadata_images_dir, filename)
    plt.savefig(filename, bbox_inches = 'tight')'''

mlid.head()


metadata



import io
import panel as pn
pn.extension()

file_input = pn.widgets.FileInput()

file_input


def transform_data(variable, window, sigma):
    """Calculates the rolling average and identifies outliers"""
    avg = metadata[variable].rolling(window=window).mean()
    residual = metadata[variable] - avg
    std = residual.rolling(window=window).std()
    outliers = np.abs(residual) > std * sigma
    return avg, avg[outliers]


def get_plot(variable="Exp", window=30, sigma=10):
    """Plots the rolling average and the outliers"""
    avg, highlight = transform_data(variable, window, sigma)
    return avg.hvplot(
        height=300, legend=False,
    ) * highlight.hvplot.scatter(padding=0.1, legend=False)


variable_widget = pn.widgets.Select(name="Target", value="Exp", options=list(metadata.columns))
window_widget = pn.widgets.IntSlider(name="window", value=30, start=1, end=60)
sigma_widget = pn.widgets.IntSlider(name="sigma", value=10, start=0, end=20)

app = pn.template.GoldenTemplate(
    site="Cyc-IF",
    title="Quality Control",
    main=[
        pn.Tabs(
            ("Dataframes", pn.Column(
                pn.Row(csv_files_button,pn.bind(handle_click, csv_files_button.param.clicks)),
                pn.pane.Markdown("### The Dataframe uploaded:"), pn.pane.DataFrame(intial_dataframe),
                #pn.pane.Markdown("### The Exposure time DataFrame is :"), pn.pane.DataFrame(exp_df.head()),
                pn.pane.Markdown("### The DataFrame after merging CycIF data x metadata :"), pn.pane.DataFrame(merged_dataframe.head()),
            )),
            ("Quality Control", pn.Column(
                quality_check(quality_control_df, not_intensities)
                #pn.pane.Markdown("### The Quality check results are:"), quality_check_results(check_shape, check_no_null, check_all_expected_files_present, check_zero_intensities)
            )),
            ("Intensities", pn.Column(
                pn.pane.Markdown("### The Not Intensities DataFrame after processing is :"), pn.pane.DataFrame(not_intensities_df, height=250),
                pn.pane.Markdown("### Select Intensities to be included"), updated_intensities,
                #pn.pane.Markdown("### The Intensities DataFrame"), intensities_df,
                #pn.pane.Markdown("### The metadata obtained that specifies the localisation:"), pn.pane.DataFrame(mlid.head())
            )),
            ("Plots", pn.Column(
                #pn.pane.Markdown(" ### Nucleus Size Distribution: "), pn.Row(nucleus_size_line_graph_with_histogram, num_of_cell_removal),
                #pn.pane.Markdown(" ### Nucleus Size Distribution: "), pn.Row(plot1,layout2),
                #pn.pane.Markdown("### Nucleus Distribution Plot:"), pn.Column(nucleus_size_plot, nucleus_size_graph),
                pn.pane.Markdown(" ### Intensity Average Plot:"), pn.Row(selected_marker_plot,num_of_cell_removal_intensity ),                
                #pn.Column(pn.Column(column_dropdown, generate_plot_button), quantile_slider, plot),
                #pn.pane.Markdown("### Cytoplasm Intensity Plot:"), cytoplasm_intensity_plot,
                #pn.pane.Markdown("### AF555_Cell_Intensity_Average:"), quantile_output_app,
                #pn.pane.Markdown("### Distribution of AF555_Cell_Intensity_Average with Quantiles:"), quantile_intensity_plot)
            )),
            
),
    ])
    
app.servable()

if __name__ == "__main__":
    pn.serve(app, port=5007)