tried to add network viz

backend/disentangle_concepts.py

@@ -80,13 +80,14 @@ def get_concepts_vectors(concepts, annotations, df, samples=100, method='LR', C=
         vectors[i,:] = vec
         important_nodes.append(set(imp_nodes))
     
-    reducer = UMAP(n_neighbors=3, # default 15, The size of local neighborhood (in terms of number of neighboring sample points) used for manifold approximation.
-                   n_components=3, # default 2, The dimension of the space to embed into.
-                   min_dist=0.1, # default 0.1, The effective minimum distance between embedded points.
-                   spread=2.0, # default 1.0, The effective scale of embedded points. In combination with ``min_dist`` this determines how clustered/clumped the embedded points are.
-                   random_state=0, # default: None, If int, random_state is the seed used by the random number generator;
-               )
+    # reducer = UMAP(n_neighbors=3, # default 15, The size of local neighborhood (in terms of number of neighboring sample points) used for manifold approximation.
+    #                n_components=3, # default 2, The dimension of the space to embed into.
+    #                min_dist=0.1, # default 0.1, The effective minimum distance between embedded points.
+    #                spread=2.0, # default 1.0, The effective scale of embedded points. In combination with ``min_dist`` this determines how clustered/clumped the embedded points are.
+    #                random_state=0, # default: None, If int, random_state is the seed used by the random number generator;
+    #            )
 
-    projection = reducer.fit_transform(vectors)
+    # projection = reducer.fit_transform(vectors)
     nodes_in_common = set.intersection(*important_nodes)
-    return vectors, projection, nodes_in_common
+    return vectors, nodes_in_common
+

pages/1_Disentanglement.py

@@ -47,7 +47,7 @@ with open(concepts) as f:
 if 'image_id' not in st.session_state:
     st.session_state.image_id = 0
 if 'concept_id' not in st.session_state:
-    st.session_state.concept_id = 'abstract'
+    st.session_state.concept_id = 'Abstract'
 
 # def on_change_random_input():
 #     st.session_state.image_id = st.session_state.image_id

pages/2_Concepts_comparison.py

@@ -1,14 +1,19 @@
 import streamlit as st
+import streamlit.components.v1 as components
+
+import pickle
+import pandas as pd
 import numpy as np
+from pyvis.network import Network
+import networkx as nx
+
+from sklearn.metrics.pairwise import cosine_similarity
 
-from plotly.subplots import make_subplots
-import plotly.graph_objects as go
+from matplotlib.backends.backend_agg import RendererAgg
 
-import graphviz
+from backend.disentangle_concepts import *
 
-#from backend.maximally_activating_patches import load_layer_infos, load_activation, get_receptive_field_coordinates
-from frontend import on_click_graph
-#from backend.utils import load_dataset_dict
+_lock = RendererAgg.lock
 
 HIGHTLIGHT_COLOR = '#e7bcc5'
 st.set_page_config(layout='wide')
@@ -19,145 +24,191 @@ st.write('> **How do the concept vectors relate to each other?**')
 st.write('> **What is their join impact on the image?**')
 st.write("""Description to write""")
 
-# -------------------------- LOAD DATASET ---------------------------------
-dataset_dict = load_dataset_dict()
-
-# -------------------------- LOAD GRAPH -----------------------------------
-
-def load_dot_to_graph(filename):
-    dot = graphviz.Source.from_file(filename)
-    source_lines = str(dot).splitlines()
-    source_lines.pop(0)
-    source_lines.pop(-1)
-    graph = graphviz.Digraph()
-    graph.body += source_lines
-    return graph, dot
-    
-
-# st.header('ConvNeXt')
-convnext_dot_file = './data/dot_architectures/convnext_architecture.dot'
-convnext_graph = load_dot_to_graph(convnext_dot_file)[0]
-
-convnext_graph.graph_attr['size'] = '4,40'
-
-# -------------------------- DISPLAY GRAPH -----------------------------------
-
-def chosen_node_text(clicked_node_title):
-    clicked_node_title = clicked_node_title.replace('stage ', 'stage_').replace('block ', 'block_')
-    stage_id = clicked_node_title.split()[0].split('_')[1] if 'stage' in clicked_node_title else None
-    block_id = clicked_node_title.split()[1].split('_')[1] if 'block' in clicked_node_title else None
-    layer_id = clicked_node_title.split()[-1]
     
-    if 'embeddings' in layer_id:
-        display_text = 'Patchify layer'
-        activation_key = 'embeddings.patch_embeddings'
-    elif 'downsampling' in layer_id:
-        display_text = f'Stage {stage_id} > Downsampling layer'
-        activation_key = f'encoder.stages[{stage_id}].downsampling_layer[1]'
-    else:
-        display_text = f'Stage {stage_id} > Block {block_id} > {layer_id} layer'
-        activation_key = f'encoder.stages[{int(stage_id)-1}].layers[{int(block_id)-1}].{layer_id}'
-    return display_text, activation_key
-
-
-props = {
-    'hightlight_color': HIGHTLIGHT_COLOR,
-    'initial_state': {
-        'group_1_header': 'Choose an option from group 1',
-        'group_2_header': 'Choose an option from group 2'
-    }
-}
-
-
-col1, col2 = st.columns((2,5))
-col1.markdown("#### Architecture")
-col1.write('')
-col1.write('Click on a layer below to generate top-k maximally activating image patches')
-col1.graphviz_chart(convnext_graph)
-
-with col2:
-    st.markdown("#### Output")
-    nodes = on_click_graph(key='toggle_buttons', **props)
-
-# -------------------------- DISPLAY OUTPUT -----------------------------------
-
-if nodes != None:
-    clicked_node_title = nodes["choice"]["node_title"]
-    clicked_node_id = nodes["choice"]["node_id"]
-    display_text, activation_key = chosen_node_text(clicked_node_title)
-    col2.write(f'**Chosen layer:** {display_text}')
-    # col2.write(f'**Activation key:** {activation_key}')
-
-    hightlight_syle = f'''
-        <style>
-            div[data-stale]:has(iframe) {{
-                height: 0;
-            }}
-            #{clicked_node_id}>polygon {{
-                fill: {HIGHTLIGHT_COLOR};
-                stroke: {HIGHTLIGHT_COLOR};
-            }}
-        </style>
-    '''
-    col2.markdown(hightlight_syle, unsafe_allow_html=True)
-
-    with col2:
-        layer_infos = None
-        with st.form('top_k_form'):
-            activation_path = './data/activation/convnext_activation.json'
-            activation = load_activation(activation_path)
-            num_channels = activation[activation_key].shape[1]
-
-            top_k = st.slider('Choose K for top-K maximally activating patches', 1,20, value=10)
-            channel_start, channel_end = st.slider(
-                'Choose channel range of this layer (recommend to choose small range less than 30)',
-                1, num_channels, value=(1, 30))
-            summit_button = st.form_submit_button('Generate image patches')
-            if summit_button:
-                
-                activation = activation[activation_key][:top_k,:,:]
-                layer_infos = load_layer_infos('./data/layer_infos/convnext_layer_infos.json')
-                # st.write(channel_start, channel_end)
-                # st.write(activation.shape, activation.shape[1])
-
-        if layer_infos != None:
-            num_cols, num_rows = top_k, channel_end - channel_start + 1
-            # num_rows = activation.shape[1]
-            top_k_coor_max_ = activation
-            st.markdown(f"#### Top-{top_k} maximally activating image patches of {num_rows} channels ({channel_start}-{channel_end})")
-
-            for row in range(channel_start, channel_end+1):
-                if row == channel_start:
-                    top_margin = 50
-                    fig = make_subplots(
-                        rows=1, cols=num_cols, 
-                        subplot_titles=tuple([f"#{i+1}" for i in range(top_k)]), shared_yaxes=True)
-                else:
-                    top_margin = 0
-                    fig = make_subplots(rows=1, cols=num_cols, shared_yaxes=True)
-                for col in range(1, num_cols+1):
-                    k, c = col-1, row-1
-                    img_index = int(top_k_coor_max_[k, c, 3])
-                    activation_value = top_k_coor_max_[k, c, 0]
-                    img = dataset_dict[img_index//10_000][img_index%10_000]['image']
-                    class_label = dataset_dict[img_index//10_000][img_index%10_000]['label']
-                    class_id = dataset_dict[img_index//10_000][img_index%10_000]['id']
-
-                    idx_x, idx_y = top_k_coor_max_[k, c, 1], top_k_coor_max_[k, c, 2]
-                    x1, x2, y1, y2 = get_receptive_field_coordinates(layer_infos, activation_key, idx_x, idx_y)
-                    img = np.array(img)[y1:y2, x1:x2, :]
-                    
-                    hovertemplate = f"""Top-{col}<br>Activation value: {activation_value:.5f}<br>Class Label: {class_label}<br>Class id: {class_id}<br>Image id: {img_index}"""
-                    fig.add_trace(go.Image(z=img, hovertemplate=hovertemplate), row=1, col=col)
-                    fig.update_xaxes(showticklabels=False, showgrid=False)
-                    fig.update_yaxes(showticklabels=False, showgrid=False)
-                    fig.update_layout(margin={'b':0, 't':top_margin, 'r':0, 'l':0})
-                    fig.update_layout(showlegend=False, yaxis_title=row)
-                    fig.update_layout(height=100, plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)')
-                    fig.update_layout(hoverlabel=dict(bgcolor="#e9f2f7"))
-                st.plotly_chart(fig, use_container_width=True)
-
-
-else:
-    col2.markdown(f'Chosen layer: <code>None</code>', unsafe_allow_html=True)
-    col2.markdown("""<style>div[data-stale]:has(iframe) {height: 0};""", unsafe_allow_html=True)
+annotations_file = './data/annotated_files/seeds0000-100000.pkl'
+with open(annotations_file, 'rb') as f:
+    annotations = pickle.load(f)
+
+ann_df = pd.read_csv('./data/annotated_files/sim_seeds0000-100000.csv')
+concepts = './data/concepts.txt'
+
+with open(concepts) as f:
+    labels = [line.strip() for line in f.readlines()]
+
+if 'image_id' not in st.session_state:
+    st.session_state.image_id = 0
+if 'concept_ids' not in st.session_state:
+    st.session_state.concept_ids = ['Abstract', 'Representational']
+
+# def on_change_random_input():
+#     st.session_state.image_id = st.session_state.image_id
+
+# ----------------------------- INPUT ----------------------------------
+st.header('Input')
+input_col_1, input_col_2, input_col_3 = st.columns(3)
+# --------------------------- INPUT column 1 ---------------------------
+with input_col_1:
+    with st.form('text_form'):
+        
+        # image_id = st.number_input('Image ID: ', format='%d', step=1)
+        st.write('**Choose a series of concepts to compare**')
+        # chosen_text_id_input = st.empty()
+        # concept_id = chosen_text_id_input.text_input('Concept:', value=st.session_state.concept_id)
+        concept_ids = st.multiselect('Concept:', tuple(labels))
+
+        choose_text_button = st.form_submit_button('Choose the defined concepts')
+        # random_text = st.form_submit_button('Select a random concept')
+
+        # if random_text:
+        #     concept_id = random.choice(labels)
+        #     st.session_state.concept_id = concept_id
+        #     chosen_text_id_input.text_input('Concept:', value=st.session_state.concept_id)
+            
+        if choose_text_button:
+            st.session_state.concept_ids = list(concept_ids)
+        # st.write(image_id, st.session_state.image_id)
+
+# ---------------------------- SET UP OUTPUT ------------------------------
+epsilon_container = st.empty()
+st.header('Output')
+st.subheader('Concept vector')
+
+# perform attack container
+# header_col_1, header_col_2, header_col_3, header_col_4, header_col_5 = st.columns([1,1,1,1,1])
+# output_col_1, output_col_2, output_col_3, output_col_4, output_col_5 = st.columns([1,1,1,1,1])
+header_col_1, header_col_2 = st.columns([5,1])
+output_col_1, output_col_2 = st.columns([5,1])
+
+st.subheader('Derivations along the concept vector')
+
+# prediction error container
+error_container = st.empty()
+smoothgrad_header_container = st.empty()
+
+# smoothgrad container
+smooth_head_1, smooth_head_2, smooth_head_3, smooth_head_4, smooth_head_5 = st.columns([1,1,1,1,1])
+smoothgrad_col_1, smoothgrad_col_2, smoothgrad_col_3, smoothgrad_col_4, smoothgrad_col_5 = st.columns([1,1,1,1,1])
+
+# ---------------------------- DISPLAY COL 1 ROW 1 ------------------------------
+with output_col_1:
+    vectors, nodes_in_common = get_concepts_vectors(concept_ids, annotations, ann_df)
+    # st.write(f'Class ID {input_id} - {input_label}: {pred_prob*100:.3f}% confidence')
+    #st.write('Concept vector', separation_vector)
+    header_col_1.write(f'Concepts {", ".join(concept_ids)} - Relevant nodes in common: {nodes_in_common}')# - Nodes {",".join(list(imp_nodes))}')
+
+    edges = []
+    for i in range(len(concepts)):
+        for j in range(len(concepts)):
+            if i != j:
+                print(f'Similarity between {concepts[i]} and {concepts[j]}')
+                similarity = cosine_similarity(vectors[i,:].reshape(1, -1), vectors[j,:].reshape(1, -1))
+                print(np.round(similarity[0][0], 3))
+                edges.append((concepts[i], concepts[j], np.round(similarity[0][0], 3)))
+
+    # Create an empty graph
+    G = nx.Graph()
+
+    # Add edges with weights to the graph
+    for edge in edges:
+        node1, node2, weight = edge
+        G.add_edge(node1, node2, weight=weight)
+
+
+    # Initiate PyVis network object
+    net = Network(
+                       height='400px',
+                       width='100%',
+                       bgcolor='#222222',
+                       font_color='white'
+                      )
+
+    # Take Networkx graph and translate it to a PyVis graph format
+    net.from_nx(G)
+
+    # Generate network with specific layout settings
+    net.repulsion(
+                        node_distance=420,
+                        central_gravity=0.33,
+                        spring_length=110,
+                        spring_strength=0.10,
+                        damping=0.95
+                       )
+
+    # Save and read graph as HTML file (on Streamlit Sharing)
+    try:
+        path = '/tmp'
+        net.save_graph(f'{path}/pyvis_graph.html')
+        HtmlFile = open(f'{path}/pyvis_graph.html', 'r', encoding='utf-8')
+
+    # Save and read graph as HTML file (locally)
+    except:
+        path = '/html_files'
+        net.save_graph(f'{path}/pyvis_graph.html')
+        HtmlFile = open(f'{path}/pyvis_graph.html', 'r', encoding='utf-8')
+
+    # Load HTML file in HTML component for display on Streamlit page
+    components.html(HtmlFile.read(), height=435)
+
+# ----------------------------- INPUT column 2 & 3 ----------------------------        
+# with input_col_2:
+#    with st.form('image_form'):
+        
+#         # image_id = st.number_input('Image ID: ', format='%d', step=1)
+#         st.write('**Choose or generate a random image to test the disentanglement**')
+#         chosen_image_id_input = st.empty()
+#         image_id = chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+        
+#         choose_image_button = st.form_submit_button('Choose the defined image')
+#         random_id = st.form_submit_button('Generate a random image')
+
+#         if random_id:
+#             image_id = random.randint(0, 100000)
+#             st.session_state.image_id = image_id
+#             chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+            
+#         if choose_image_button:
+#             image_id = int(image_id)
+#             st.session_state.image_id = int(image_id)
+#         # st.write(image_id, st.session_state.image_id)
+
+# with input_col_3:
+#     with st.form('Variate along the disentangled concept'):
+#         st.write('**Set range of change**')
+#         chosen_epsilon_input = st.empty()
+#         epsilon = chosen_epsilon_input.number_input('Epsilon:', min_value=1, step=1)
+#         epsilon_button = st.form_submit_button('Choose the defined epsilon')
+
+# # ---------------------------- DISPLAY COL 2 ROW 1 ------------------------------
+
+# #model = torch.load('./data/model_files/pytorch_model.bin', map_location=torch.device('cpu'))
+# with dnnlib.util.open_url('./data/model_files/network-snapshot-010600.pkl') as f:
+#     model = legacy.load_network_pkl(f)['G_ema'].to('cpu') # type: ignore
+
+# original_image_vec = annotations['z_vectors'][st.session_state.image_id]
+# img = generate_original_image(original_image_vec, model)
+# # input_image = original_image_dict['image']
+# # input_label = original_image_dict['label']
+# # input_id = original_image_dict['id']
+
+# with smoothgrad_col_3:
+#     st.image(img)
+#     smooth_head_3.write(f'Base image')
+
+
+# images, lambdas = regenerate_images(model, original_image_vec, separation_vector, min_epsilon=-(int(epsilon)), max_epsilon=int(epsilon))
+
+# with smoothgrad_col_1:
+#     st.image(images[0])
+#     smooth_head_1.write(f'Change of {np.round(lambdas[0], 2)}')
+
+# with smoothgrad_col_2:
+#     st.image(images[1])
+#     smooth_head_2.write(f'Change of {np.round(lambdas[1], 2)}')
+
+# with smoothgrad_col_4:
+#     st.image(images[3])
+#     smooth_head_4.write(f'Change of {np.round(lambdas[3], 2)}')
+
+# with smoothgrad_col_5:
+#     st.image(images[4])
+#     smooth_head_5.write(f'Change of {np.round(lambdas[4], 2)}')

requirements.txt

@@ -16,3 +16,6 @@ altair==4.0
 #torch-utils
 opencv-python
 umap-learn
+graphviz
+networkx
+pyvis