Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.cluster import KMeans | |
| from sklearn.metrics.pairwise import linear_kernel, cosine_similarity | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk import FreqDist | |
| import re | |
| import os | |
| import base64 | |
| from graphviz import Digraph | |
| from io import BytesIO | |
| import networkx as nx | |
| import matplotlib.pyplot as plt | |
| st.set_page_config( | |
| page_title="📺Transcript📜EDA🔍NLTK", | |
| page_icon="🌠", | |
| layout="wide", | |
| initial_sidebar_state="expanded", | |
| menu_items={ | |
| 'Get Help': 'https://huggingface.co/awacke1', | |
| 'Report a bug': "https://huggingface.co/awacke1", | |
| 'About': "https://huggingface.co/awacke1" | |
| } | |
| ) | |
| st.markdown(''' | |
| 1. 🔍 **Transcript Insights Using Exploratory Data Analysis (EDA)** 📊 - Unveil hidden patterns 🕵️♂️ and insights 🧠 in your transcripts. 🏆. | |
| 2. 📜 **Natural Language Toolkit (NLTK)** 🛠️:- your compass 🧭 in the vast landscape of NLP. | |
| 3. 📺 **Transcript Analysis** 📈:Speech recognition 🎙️ and thematic extraction 🌐, audiovisual content to actionable insights 🔑. | |
| ''') | |
| # 📥 Download NLTK data | |
| def download_nltk_data(): | |
| try: | |
| nltk.data.find('tokenizers/punkt') | |
| nltk.data.find('corpora/stopwords') | |
| except LookupError: | |
| with st.spinner('Downloading required NLTK data...'): | |
| nltk.download('punkt') | |
| nltk.download('stopwords') | |
| st.success('NLTK data is ready!') | |
| download_nltk_data() | |
| # 🕰️ Remove timestamps | |
| def remove_timestamps(text): | |
| return re.sub(r'\d{1,2}:\d{2}\n.*\n', '', text) | |
| # 📊 Extract high information words | |
| def extract_high_information_words(text, top_n=10): | |
| try: | |
| words = [word.lower() for word in nltk.word_tokenize(text) if word.isalpha()] | |
| stop_words = set(stopwords.words('english')) | |
| filtered_words = [word for word in words if word not in stop_words] | |
| return [word for word, _ in FreqDist(filtered_words).most_common(top_n)] | |
| except Exception as e: | |
| st.error(f"Error in extract_high_information_words: {str(e)}") | |
| return [] | |
| # 🔗 Create relationship graph | |
| def create_relationship_graph(words): | |
| graph = Digraph() | |
| for i, word in enumerate(words): | |
| graph.node(str(i), word) | |
| if i > 0: | |
| graph.edge(str(i-1), str(i), label=word) | |
| return graph | |
| # 📈 Display relationship graph | |
| def display_relationship_graph(words): | |
| try: | |
| graph = create_relationship_graph(words) | |
| st.graphviz_chart(graph) | |
| except Exception as e: | |
| st.error(f"Error displaying relationship graph: {str(e)}") | |
| # 🔍 Extract context words | |
| def extract_context_words(text, high_information_words): | |
| words = nltk.word_tokenize(text) | |
| return [(words[i-1] if i > 0 else None, word, words[i+1] if i < len(words)-1 else None) | |
| for i, word in enumerate(words) if word.lower() in high_information_words] | |
| # 📊 Create context graph | |
| def create_context_graph(context_words): | |
| graph = Digraph() | |
| for i, (before, high, after) in enumerate(context_words): | |
| if before: | |
| graph.node(f'before{i}', before, shape='box') | |
| graph.edge(f'before{i}', f'high{i}', label=before) | |
| graph.node(f'high{i}', high, shape='ellipse') | |
| if after: | |
| graph.node(f'after{i}', after, shape='diamond') | |
| graph.edge(f'high{i}', f'after{i}', label=after) | |
| return graph | |
| # 📈 Display context graph | |
| def display_context_graph(context_words): | |
| try: | |
| graph = create_context_graph(context_words) | |
| st.graphviz_chart(graph) | |
| except Exception as e: | |
| st.error(f"Error displaying context graph: {str(e)}") | |
| # 📊 Display context table | |
| def display_context_table(context_words): | |
| table = "| Before | High Info Word | After |\n|--------|----------------|-------|\n" | |
| table += "\n".join(f"| {b if b else ''} | {h} | {a if a else ''} |" for b, h, a in context_words) | |
| st.markdown(table) | |
| # 📁 Load example files | |
| def load_example_files(): | |
| excluded_files = {'freeze.txt', 'requirements.txt', 'packages.txt', 'pre-requirements.txt'} | |
| example_files = [f for f in os.listdir() if f.endswith('.txt') and f not in excluded_files] | |
| if example_files: | |
| selected_file = st.selectbox("📄 Select an example file:", example_files) | |
| if st.button(f"📂 Load {selected_file}"): | |
| with open(selected_file, 'r', encoding="utf-8") as file: | |
| return file.read() | |
| else: | |
| st.write("No suitable example files found.") | |
| return None | |
| # 🧠 Cluster sentences | |
| def cluster_sentences(sentences, num_clusters): | |
| sentences = [s for s in sentences if len(s) > 10] | |
| num_clusters = min(num_clusters, len(sentences)) | |
| vectorizer = TfidfVectorizer() | |
| X = vectorizer.fit_transform(sentences) | |
| kmeans = KMeans(n_clusters=num_clusters, random_state=42) | |
| kmeans.fit(X) | |
| clustered_sentences = [[] for _ in range(num_clusters)] | |
| for i, label in enumerate(kmeans.labels_): | |
| similarity = linear_kernel(kmeans.cluster_centers_[label:label+1], X[i:i+1]).flatten()[0] | |
| clustered_sentences[label].append((similarity, sentences[i])) | |
| return [[s for _, s in sorted(cluster, reverse=True)] for cluster in clustered_sentences] | |
| # 💾 Get text file download link | |
| def get_text_file_download_link(text_to_download, filename='Output.txt', button_label="💾 Save"): | |
| b64 = base64.b64encode(text_to_download.encode()).decode() | |
| return f'<a href="data:file/txt;base64,{b64}" download="{filename}" style="margin-top:20px;">{button_label}</a>' | |
| # 📊 Get high info words per cluster | |
| def get_high_info_words_per_cluster(cluster_sentences, num_words=5): | |
| return [extract_high_information_words(" ".join(cluster), num_words) for cluster in cluster_sentences] | |
| # 📊 Plot cluster words | |
| def plot_cluster_words(cluster_sentences): | |
| for i, cluster in enumerate(cluster_sentences): | |
| words = re.findall(r'\b[a-z]{4,}\b', " ".join(cluster)) | |
| word_freq = FreqDist(words) | |
| top_words = [word for word, _ in word_freq.most_common(20)] | |
| vectorizer = TfidfVectorizer() | |
| X = vectorizer.fit_transform(top_words) | |
| similarity_matrix = cosine_similarity(X.toarray()) | |
| G = nx.from_numpy_array(similarity_matrix) | |
| pos = nx.spring_layout(G, k=0.5) | |
| plt.figure(figsize=(8, 6)) | |
| nx.draw_networkx(G, pos, node_size=500, font_size=12, font_weight='bold', with_labels=True, | |
| labels={i: word for i, word in enumerate(top_words)}, | |
| node_color='skyblue', edge_color='gray') | |
| plt.axis('off') | |
| plt.title(f"Cluster {i+1} Word Arrangement") | |
| st.pyplot(plt) | |
| st.markdown(f"**Cluster {i+1} Details:**") | |
| st.markdown(f"Top Words: {', '.join(top_words)}") | |
| st.markdown(f"Number of Sentences: {len(cluster)}") | |
| st.markdown("---") | |
| # Main code for UI | |
| uploaded_file = st.file_uploader("📁 Choose a .txt file", type=['txt']) | |
| example_text = load_example_files() | |
| if example_text: | |
| file_text = example_text | |
| elif uploaded_file: | |
| file_text = uploaded_file.read().decode("utf-8") | |
| else: | |
| file_text = "" | |
| if file_text: | |
| text_without_timestamps = remove_timestamps(file_text) | |
| top_words = extract_high_information_words(text_without_timestamps, 10) | |
| with st.expander("📊 Top 10 High Information Words"): | |
| st.write(top_words) | |
| with st.expander("📈 Relationship Graph"): | |
| display_relationship_graph(top_words) | |
| context_words = extract_context_words(text_without_timestamps, top_words) | |
| with st.expander("🔗 Context Graph"): | |
| display_context_graph(context_words) | |
| with st.expander("📑 Context Table"): | |
| display_context_table(context_words) | |
| sentences = [line.strip() for line in file_text.split('\n') if len(line.strip()) > 10] | |
| num_sentences = len(sentences) | |
| st.write(f"Total Sentences: {num_sentences}") | |
| num_clusters = st.slider("Number of Clusters", min_value=2, max_value=10, value=5) | |
| clustered_sentences = cluster_sentences(sentences, num_clusters) | |
| col1, col2 = st.columns(2) | |
| with col1: | |
| st.subheader("Original Text") | |
| original_text = "\n".join(sentences) | |
| st.text_area("Original Sentences", value=original_text, height=400) | |
| with col2: | |
| st.subheader("Clustered Text") | |
| clusters = "" | |
| clustered_text = "" | |
| cluster_high_info_words = get_high_info_words_per_cluster(clustered_sentences) | |
| for i, cluster in enumerate(clustered_sentences): | |
| cluster_text = "\n".join(cluster) | |
| high_info_words = ", ".join(cluster_high_info_words[i]) | |
| clusters += f"Cluster {i+1} (High Info Words: {high_info_words})\n" | |
| clustered_text += f"Cluster {i+1} (High Info Words: {high_info_words}):\n{cluster_text}\n\n" | |
| st.text_area("Clusters", value=clusters, height=200) | |
| st.text_area("Clustered Sentences", value=clustered_text, height=200) | |
| clustered_sentences_flat = [sentence for cluster in clustered_sentences for sentence in cluster] | |
| if set(sentences) == set(clustered_sentences_flat): | |
| st.write("✅ All sentences are accounted for in the clustered output.") | |
| else: | |
| st.write("❌ Some sentences are missing in the clustered output.") | |
| plot_cluster_words(clustered_sentences) | |
| st.markdown("For more information and updates, visit our [help page](https://huggingface.co/awacke1).") |