Upload 4 files

analysis.py

@@ -0,0 +1,69 @@
+import os
+import re
+import json
+import networkx as nx
+import matplotlib.pyplot as plt
+
+# ---------------- DATA EXTRACTION ----------------
+
+def parse_page(page_name):
+    """Extract model and lora names from the given HTML page."""
+    with open(page_name, 'r', encoding='utf-8') as f:
+        html_content = f.read()
+    return extract_data_from_html(html_content)
+
+def extract_data_from_html(html_content):
+    """Extract specific data from the embedded JSON within the HTML content."""
+    pattern = r'id="__NEXT_DATA__" type="application/json">(.*?)</script><script defer'
+    match = re.search(pattern, html_content)
+    if not match:
+        return None
+    
+    json_string = match.group(1)
+    data_dict = json.loads(json_string)
+    return get_model_and_resources(data_dict)
+
+def get_model_and_resources(data_dict):
+    """Retrieve model and associated resources from the parsed JSON data."""
+    model_name = data_dict['props']['pageProps']['trpcState']['json']['queries'][0]['state']['data']['meta']['Model']
+    lora_names = [r['name'] for r in data_dict['props']['pageProps']['trpcState']['json']['queries'][0]['state']['data']['meta']['resources'] if r['type'] == 'lora']
+    return model_name, lora_names
+
+# ---------------- GRAPH CONSTRUCTION ----------------
+
+def build_graph(data, degree_threshold):
+    """Build a bipartite graph from the data and prune nodes with degrees below the threshold."""
+    B = nx.Graph()
+    
+    for page_name, (model, loras) in data.items():
+        B.add_node(model, bipartite=0)
+        for lora in loras:
+            B.add_node(lora, bipartite=1)
+        B.add_edge(model, lora, page=page_name.split('.')[0])
+    
+    nodes_to_remove = [node for node, degree in dict(B.degree()).items() if degree < degree_threshold]
+    B.remove_nodes_from(nodes_to_remove)
+    return B
+
+
+# ---------------- VISUALIZATION AND ANALYSIS ----------------
+
+def visualize_bipartite(B):
+    """Visualize the bipartite graph."""
+    model_nodes = {n for n, d in B.nodes(data=True) if d['bipartite']==0}
+    lora_nodes = set(B) - model_nodes
+    pos = nx.bipartite_layout(B, model_nodes)
+    
+    plt.figure(figsize=(10, 5))
+    nx.draw(B, pos, with_labels=True, node_color=['#1f78b4' if node in model_nodes else '#33a02c' for node in B.nodes()])
+    plt.title("Bipartite Graph between Model Name and Lora Name")
+    plt.show()
+
+def most_connected_models(B, top_n=10):
+    """List the most connected models in the bipartite graph."""
+    model_nodes = {n for n, d in B.nodes(data=True) if d['bipartite']==0}
+    sorted_models = sorted(model_nodes, key=lambda x: B.degree(x), reverse=True)
+    
+    for model in sorted_models[:top_n]:
+        loras = list(B.neighbors(model))
+        print(f"Model: {model}, Connected Loras: {loras}")

app.py

@@ -0,0 +1,91 @@
+import streamlit as st
+import pandas as pd
+import networkx as nx
+import os
+import pickle
+import tqdm
+from analysis import build_graph, parse_page
+
+def clean(results):
+    new = {}
+    for k in results:
+        if results[k] and len(results[k]) > 0:
+            new[k] = results[k]
+    return new
+
+# Your existing functions here...
+if "B_degree_threshold" not in st.session_state:
+    st.session_state.B_degree_threshold = 10
+if "B" not in st.session_state:
+    if not os.path.exists('data.pkl'):
+        page_folder = 'pages'
+        pages = os.listdir(page_folder)
+        results = {}
+        
+        for p in tqdm.tqdm(pages):
+            try:
+                results[p] = parse_page(os.path.join(page_folder, p))
+            except Exception as e:
+                pass
+        
+        with open('data.pkl', 'wb') as f:
+            pickle.dump(results, f)
+
+    else:
+        with open('data.pkl', 'rb') as f:
+            results = pickle.load(f)
+    
+    st.session_state.results = clean(results)
+    st.session_state.B = build_graph(st.session_state.results, st.session_state.B_degree_threshold)
+
+
+
+# Streamlit app
+def main():
+    st.title("Model-Lora Relationship Viewer")
+
+    # Sidebar for degree_threshold
+    B_degree_threshold = st.sidebar.slider("Select Degree Threshold", 1, 100, 10)
+
+    # Build the graph
+    if B_degree_threshold != st.session_state.B_degree_threshold:
+        st.session_state.B_degree_threshold = B_degree_threshold
+        st.session_state.B = build_graph(st.session_state.results, B_degree_threshold)
+
+    st.sidebar.write(f"There are {len(st.session_state.B)} node analyzed.")
+
+    # Filter out model nodes and lora nodes
+    model_nodes = {n for n, d in st.session_state.B.nodes(data=True) if d['bipartite']==0}
+    lora_nodes = set(st.session_state.B) - model_nodes
+
+    # Sort model nodes and lora nodes based on their degree
+    sorted_models = sorted(model_nodes, key=lambda x: st.session_state.B.degree(x), reverse=True)
+    sorted_loras = sorted(lora_nodes, key=lambda x: st.session_state.B.degree(x), reverse=True)
+
+    # Model selection
+    selected_model = st.selectbox("Select Model (sorted by degree)", sorted_models)
+    if selected_model:
+        loras_for_model = list(st.session_state.B.neighbors(selected_model))
+        page_names_for_model = [st.session_state.B[selected_model][lora]['page'] for lora in loras_for_model]
+        page_names_for_model = ['https://civitai.com/images/'+page for page in page_names_for_model]
+        
+        # Convert DataFrame to HTML with clickable links
+        df = pd.DataFrame({"Lora Names": loras_for_model, "Image Link": page_names_for_model})
+        df["Image Link"] = df["Image Link"].apply(lambda x: f'<a href="{x}" target="_blank">{x}</a>')
+        st.markdown(df.to_html(escape=False, index=False), unsafe_allow_html=True)
+
+    # Lora selection
+    selected_lora = st.selectbox("Select Lora (sorted by degree)", sorted_loras)
+    if selected_lora:
+        models_for_lora = list(st.session_state.B.neighbors(selected_lora))
+        page_names_for_lora = [st.session_state.B[model][selected_lora]['page'] for model in models_for_lora]
+        page_names_for_lora = ['https://civitai.com/images/'+page for page in page_names_for_lora]
+        
+        # Convert DataFrame to HTML with clickable links
+        df = pd.DataFrame({"Model Names": models_for_lora, "Image Link": page_names_for_lora})
+        df["Image Link"] = df["Image Link"].apply(lambda x: f'<a href="{x}" target="_blank">{x}</a>')
+        st.markdown(df.to_html(escape=False, index=False), unsafe_allow_html=True)
+
+
+if __name__ == "__main__":
+    main()

data.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ab9484f2c5b8fa2cb8fdccc4aeb559da173a8e4de2c325fe8f7286544f8d528
+size 2856767

requirements.txt

@@ -0,0 +1,7 @@
+matplotlib==3.7.0
+networkx==3.0
+pandas==1.5.3
+requests==2.28.2
+retrying==1.3.4
+streamlit==1.25.0
+tqdm==4.64.1