Spaces:
Sleeping
Sleeping
| # Import necessary libraries | |
| import streamlit as st | |
| import re | |
| import nltk | |
| import os | |
| from nltk.corpus import stopwords | |
| from nltk import FreqDist | |
| from graphviz import Digraph | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.cluster import KMeans | |
| # Set page configuration with a title and favicon | |
| st.set_page_config( | |
| page_title="📺Transcript📜EDA🔍NLTK", | |
| page_icon="🌠", | |
| layout="wide", | |
| initial_sidebar_state="expanded", | |
| menu_items={ | |
| 'Get Help': 'https://huggingface.co/awacke1', | |
| 'Report a bug': "https://huggingface.co/spaces/awacke1/WebDataDownload", | |
| 'About': "# Midjourney: https://discord.com/channels/@me/997514686608191558" | |
| } | |
| ) | |
| st.markdown('''🔍 **Exploratory Data Analysis (EDA)** 📊: - Dive deep into the sea of data with our EDA feature, unveiling hidden patterns 🕵️♂️ and insights 🧠 in your transcripts. Transform raw data into a treasure trove of information 🏆. | |
| 📜 **Natural Language Toolkit (NLTK)** 🛠️: - Harness the power of NLTK to process and understand human language 🗣️. From tokenization to sentiment analysis, our toolkit is your compass 🧭 in the vast landscape of natural language processing (NLP). | |
| 📺 **Transcript Analysis** 📈: - Elevate your text analysis with our advanced transcript analysis tools. Whether it's speech recognition 🎙️ or thematic extraction 🌐, turn your audiovisual content into actionable insights 🔑.''') | |
| # Download NLTK resources | |
| nltk.download('punkt') | |
| nltk.download('stopwords') | |
| def remove_timestamps(text): | |
| return re.sub(r'\d{1,2}:\d{2}\n.*\n', '', text) | |
| def extract_high_information_words(text, top_n=10): | |
| words = nltk.word_tokenize(text) | |
| words = [word.lower() for word in words if word.isalpha()] | |
| stop_words = set(stopwords.words('english')) | |
| filtered_words = [word for word in words if word not in stop_words] | |
| freq_dist = FreqDist(filtered_words) | |
| return [word for word, _ in freq_dist.most_common(top_n)] | |
| def create_relationship_graph(words): | |
| graph = Digraph() | |
| for index, word in enumerate(words): | |
| graph.node(str(index), word) | |
| if index > 0: | |
| graph.edge(str(index - 1), str(index), label=str(index)) | |
| return graph | |
| def display_relationship_graph(words): | |
| graph = create_relationship_graph(words) | |
| st.graphviz_chart(graph) | |
| def extract_context_words(text, high_information_words): | |
| words = nltk.word_tokenize(text) | |
| context_words = [] | |
| for index, word in enumerate(words): | |
| if word.lower() in high_information_words: | |
| before_word = words[index - 1] if index > 0 else None | |
| after_word = words[index + 1] if index < len(words) - 1 else None | |
| context_words.append((before_word, word, after_word)) | |
| return context_words | |
| def create_context_graph(context_words): | |
| graph = Digraph() | |
| for index, (before_word, high_info_word, after_word) in enumerate(context_words): | |
| #graph.node(f'before{index}', before_word, shape='box') if before_word else None | |
| if before_word: graph.node(f'before{index}', before_word, shape='box') # else None | |
| graph.node(f'high{index}', high_info_word, shape='ellipse') | |
| #graph.node(f'after{index}', after_word, shape='diamond') if after_word else None | |
| if after_word: graph.node(f'after{index}', after_word, shape='diamond') # else None | |
| if before_word: | |
| graph.edge(f'before{index}', f'high{index}') | |
| if after_word: | |
| graph.edge(f'high{index}', f'after{index}') | |
| return graph | |
| def display_context_graph(context_words): | |
| graph = create_context_graph(context_words) | |
| st.graphviz_chart(graph) | |
| def display_context_table(context_words): | |
| table = "| Before | High Info Word | After |\n|--------|----------------|-------|\n" | |
| for before, high, after in context_words: | |
| table += f"| {before if before else ''} | {high} | {after if after else ''} |\n" | |
| st.markdown(table) | |
| def load_example_files(): | |
| # Exclude specific files | |
| excluded_files = {'freeze.txt', 'requirements.txt', 'packages.txt', 'pre-requirements.txt'} | |
| # List all .txt files excluding the ones in excluded_files | |
| example_files = [f for f in os.listdir() if f.endswith('.txt') and f not in excluded_files] | |
| # Check if there are any files to select from | |
| if example_files: | |
| selected_file = st.selectbox("📄 Select an example file:", example_files) | |
| if st.button(f"📂 Load {selected_file}"): | |
| with open(selected_file, 'r', encoding="utf-8") as file: | |
| return file.read() | |
| else: | |
| st.write("No suitable example files found.") | |
| return None | |
| def cluster_sentences(sentences, num_clusters): | |
| # Filter sentences with length over 10 characters | |
| sentences = [sentence for sentence in sentences if len(sentence) > 10] | |
| # Check if the number of sentences is less than the desired number of clusters | |
| if len(sentences) < num_clusters: | |
| # If so, adjust the number of clusters to match the number of sentences | |
| num_clusters = len(sentences) | |
| # Vectorize the sentences | |
| vectorizer = TfidfVectorizer() | |
| X = vectorizer.fit_transform(sentences) | |
| # Perform k-means clustering | |
| kmeans = KMeans(n_clusters=num_clusters, random_state=42) | |
| kmeans.fit(X) | |
| # Get the cluster labels for each sentence | |
| labels = kmeans.labels_ | |
| # Group sentences by cluster | |
| clustered_sentences = [[] for _ in range(num_clusters)] | |
| for i, label in enumerate(labels): | |
| clustered_sentences[label].append((i, sentences[i])) | |
| return clustered_sentences | |
| # Main code for UI | |
| uploaded_file = st.file_uploader("📁 Choose a .txt file", type=['txt']) | |
| example_text = load_example_files() | |
| if example_text: | |
| file_text = example_text | |
| elif uploaded_file: | |
| file_text = uploaded_file.read().decode("utf-8") | |
| else: | |
| file_text = "" | |
| if file_text: | |
| text_without_timestamps = remove_timestamps(file_text) | |
| sentences = [sentence.strip() for sentence in text_without_timestamps.split('.') if len(sentence.strip()) > 10] | |
| top_words = extract_high_information_words(text_without_timestamps, 10) | |
| with st.expander("📊 Top 10 High Information Words"): | |
| st.write(top_words) | |
| with st.expander("📈 Relationship Graph"): | |
| display_relationship_graph(top_words) | |
| context_words = extract_context_words(text_without_timestamps, top_words) | |
| with st.expander("🔗 Context Graph"): | |
| display_context_graph(context_words) | |
| with st.expander("📑 Context Table"): | |
| display_context_table(context_words) | |
| # with st.expander("Innovation Outlines"): | |
| # showInnovationOutlines() | |
| with st.expander("📝 Sentence Clustering"): | |
| num_clusters = st.slider("Number of Clusters", min_value=2, max_value=10, value=5) | |
| clustered_sentences = cluster_sentences(sentences, num_clusters) | |
| output_text = "" | |
| for i, cluster in enumerate(clustered_sentences): | |
| output_text += f"## 🌟 Cluster {i+1}:\n" | |
| for original_index, sentence in cluster: | |
| output_text += f"- Original Line {original_index+1}: {sentence}\n" | |
| output_text += "\n" | |
| st.markdown(output_text) | |
| st.markdown("https://cdn-uploads.huggingface.co/production/uploads/620630b603825909dcbeba35/Id9kntHFHZf_oFFrEmGh5.png") | |