Update app.py

app.py

@@ -1,3 +1,4 @@
+# Import necessary libraries
 import streamlit as st
 import re
 import nltk
@@ -7,9 +8,6 @@ from nltk import FreqDist
 from graphviz import Digraph
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.cluster import KMeans
-from sklearn.metrics.pairwise import linear_kernel
-from io import BytesIO
-import base64
 
 # Set page configuration with a title and favicon
 st.set_page_config(
@@ -24,6 +22,10 @@ st.set_page_config(
     }
 )
 
+st.markdown('''🔍 **Exploratory Data Analysis (EDA)** 📊:     - Dive deep into the sea of data with our EDA feature, unveiling hidden patterns 🕵️‍♂️ and insights 🧠 in your transcripts. Transform raw data into a treasure trove of information 🏆.
+📜 **Natural Language Toolkit (NLTK)** 🛠️:     - Harness the power of NLTK to process and understand human language 🗣️. From tokenization to sentiment analysis, our toolkit is your compass 🧭 in the vast landscape of natural language processing (NLP).
+📺 **Transcript Analysis** 📈:     - Elevate your text analysis with our advanced transcript analysis tools. Whether it's speech recognition 🎙️ or thematic extraction 🌐, turn your audiovisual content into actionable insights 🔑.''')
+
 # Download NLTK resources
 nltk.download('punkt')
 nltk.download('stopwords')
@@ -39,9 +41,76 @@ def extract_high_information_words(text, top_n=10):
     freq_dist = FreqDist(filtered_words)
     return [word for word, _ in freq_dist.most_common(top_n)]
 
+def create_relationship_graph(words):
+    graph = Digraph()
+    for index, word in enumerate(words):
+        graph.node(str(index), word)
+        if index > 0:
+            graph.edge(str(index - 1), str(index), label=str(index))
+    return graph
+
+def display_relationship_graph(words):
+    graph = create_relationship_graph(words)
+    st.graphviz_chart(graph)
+
+def extract_context_words(text, high_information_words):
+    words = nltk.word_tokenize(text)
+    context_words = []
+    for index, word in enumerate(words):
+        if word.lower() in high_information_words:
+            before_word = words[index - 1] if index > 0 else None
+            after_word = words[index + 1] if index < len(words) - 1 else None
+            context_words.append((before_word, word, after_word))
+    return context_words
+
+def create_context_graph(context_words):
+    graph = Digraph()
+    for index, (before_word, high_info_word, after_word) in enumerate(context_words):
+        #graph.node(f'before{index}', before_word, shape='box') if before_word else None
+        if before_word:  graph.node(f'before{index}', before_word, shape='box') # else None
+        graph.node(f'high{index}', high_info_word, shape='ellipse')
+        #graph.node(f'after{index}', after_word, shape='diamond') if after_word else None
+        if after_word: graph.node(f'after{index}', after_word, shape='diamond') # else None
+        if before_word:
+            graph.edge(f'before{index}', f'high{index}')
+        if after_word:
+            graph.edge(f'high{index}', f'after{index}')
+    return graph
+
+def display_context_graph(context_words):
+    graph = create_context_graph(context_words)
+    st.graphviz_chart(graph)
+
+def display_context_table(context_words):
+    table = "| Before | High Info Word | After |\n|--------|----------------|-------|\n"
+    for before, high, after in context_words:
+        table += f"| {before if before else ''} | {high} | {after if after else ''} |\n"
+    st.markdown(table)
+
+
+def load_example_files():
+    # Exclude specific files
+    excluded_files = {'freeze.txt', 'requirements.txt', 'packages.txt', 'pre-requirements.txt'}
+    
+    # List all .txt files excluding the ones in excluded_files
+    example_files = [f for f in os.listdir() if f.endswith('.txt') and f not in excluded_files]
+    
+    # Check if there are any files to select from
+    if example_files:
+        selected_file = st.selectbox("📄 Select an example file:", example_files)
+        if st.button(f"📂 Load {selected_file}"):
+            with open(selected_file, 'r', encoding="utf-8") as file:
+                return file.read()
+    else:
+        st.write("No suitable example files found.")
+    
+    return None
+
 def cluster_sentences(sentences, num_clusters):
-    # Filter sentences with length over 10 characters
-    sentences = [sentence for sentence in sentences if len(sentence) > 10]
+    # Check if the number of sentences is less than the desired number of clusters
+    if len(sentences) < num_clusters:
+        # If so, adjust the number of clusters to match the number of sentences
+        num_clusters = len(sentences)
 
     # Vectorize the sentences
     vectorizer = TfidfVectorizer()
@@ -51,56 +120,59 @@ def cluster_sentences(sentences, num_clusters):
     kmeans = KMeans(n_clusters=num_clusters, random_state=42)
     kmeans.fit(X)
 
-    # Calculate the centroid of each cluster
-    cluster_centers = kmeans.cluster_centers_
+    # Get the cluster labels for each sentence
+    labels = kmeans.labels_
 
-    # Group sentences by cluster and calculate similarity to centroid
+    # Group sentences by cluster
     clustered_sentences = [[] for _ in range(num_clusters)]
-    for i, label in enumerate(kmeans.labels_):
-        similarity = linear_kernel(cluster_centers[label:label+1], X[i:i+1]).flatten()[0]
-        clustered_sentences[label].append((similarity, sentences[i]))
-
-    # Order sentences within each cluster based on their similarity to the centroid
-    for cluster in clustered_sentences:
-        cluster.sort(reverse=True)  # Sort based on similarity (descending order)
-
-    # Return the ordered clustered sentences without similarity scores for display
-    return [[sentence for _, sentence in cluster] for cluster in clustered_sentences]
-
-# Function to convert text to a downloadable file
-def get_text_file_download_link(text_to_download, filename='Output.txt', button_label="💾 Save"):
-    buffer = BytesIO()
-    buffer.write(text_to_download.encode())
-    buffer.seek(0)
-    b64 = base64.b64encode(buffer.read()).decode()
-    href = f'<a href="data:file/txt;base64,{b64}" download="{filename}" style="margin-top:20px;">{button_label}</a>'
-    return href
+    for i, label in enumerate(labels):
+        clustered_sentences[label].append(sentences[i])
+
+    return clustered_sentences
 
 # Main code for UI
 uploaded_file = st.file_uploader("📁 Choose a .txt file", type=['txt'])
 
-if uploaded_file:
+example_text = load_example_files()
+
+if example_text:
+    file_text = example_text
+elif uploaded_file:
     file_text = uploaded_file.read().decode("utf-8")
 else:
     file_text = ""
 
 if file_text:
     text_without_timestamps = remove_timestamps(file_text)
-    sentences = [sentence.strip() for sentence in text_without_timestamps.split('.') if len(sentence.strip()) > 10]
+    top_words = extract_high_information_words(text_without_timestamps, 10)
+
+    with st.expander("📊 Top 10 High Information Words"):
+        st.write(top_words)
+
+    with st.expander("📈 Relationship Graph"):
+        display_relationship_graph(top_words)
+
+    context_words = extract_context_words(text_without_timestamps, top_words)
+
+    with st.expander("🔗 Context Graph"):
+        display_context_graph(context_words)
+
+    with st.expander("📑 Context Table"):
+        display_context_table(context_words)
+
+    #with st.expander("Innovation Outlines"):
+    #    showInnovationOutlines()
 
     with st.expander("📝 Sentence Clustering"):
+        sentences = [sentence.strip() for sentence in text_without_timestamps.split('.') if sentence.strip()]
+
         num_clusters = st.slider("Number of Clusters", min_value=2, max_value=10, value=5)
         clustered_sentences = cluster_sentences(sentences, num_clusters)
 
+        output_text = ""
         for i, cluster in enumerate(clustered_sentences):
-            st.text_area(f"Cluster {i+1}", value="\n".join(cluster), height=100)
-            
-            # Input for custom filename
-            default_filename = f"Cluster_{i+1}_Output.txt"
-            filename = st.text_input("Enter filename for download:", value=default_filename, key=f"filename_{i}")
-            
-            # Download button
-            download_link = get_text_file_download_link("\n".join(cluster), filename, f"💾 Save Cluster {i+1}")
-            st.markdown(download_link, unsafe_allow_html=True)
-
-st.markdown("For more information and updates, visit our [help page](https://huggingface.co/awacke1).")
+            output_text += f"Cluster {i+1}:\n"
+            output_text += "\n".join(cluster)
+            output_text += "\n\n"
+
+        st.text_area("Clustered Sentences", value=output_text, height=400)