Spaces:
Sleeping
Sleeping
| import nltk | |
| nltk.download('stopwords') | |
| nltk.download('wordnet') | |
| from nltk.corpus import stopwords | |
| from nltk.stem.wordnet import WordNetLemmatizer | |
| import gensim | |
| from gensim.utils import simple_preprocess | |
| import re | |
| import pyLDAvis | |
| import pyLDAvis.gensim | |
| import pandas as pd | |
| import re | |
| import gensim.corpora as corpora | |
| import spacy | |
| import os | |
| import gradio as gr | |
| import spacy.cli | |
| spacy.cli.download("en_core_web_sm") | |
| def preprocess_response(response): | |
| # Remove punctuation | |
| response = re.sub(r'[,\.!?]', '', response) | |
| # Convert to lowercase | |
| response = response.lower() | |
| return response | |
| def preprocess_responses(responses): | |
| preprocessed_responses = [preprocess_response(response) for response in responses] | |
| return preprocessed_responses | |
| def sent_to_words(sentences): | |
| for sentence in sentences: | |
| yield(gensim.utils.simple_preprocess(str(sentence), deacc=True)) # deacc=True removes punctuations | |
| def make_bigram_trigram_models(data_words): | |
| # Build the bigram and trigram models | |
| bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold fewer phrases. | |
| trigram = gensim.models.Phrases(bigram[data_words], threshold=100) | |
| # Faster way to get a sentence clubbed as a trigram/bigram | |
| bigram_mod = gensim.models.phrases.Phraser(bigram) | |
| trigram_mod = gensim.models.phrases.Phraser(trigram) | |
| return bigram_mod, trigram_mod | |
| def make_bigrams(texts, bigram_mod): | |
| return [bigram_mod[doc] for doc in texts] | |
| def remove_stopwords(texts, stop_words): | |
| return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts] | |
| def lemmatization(texts, nlp, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']): | |
| """https://spacy.io/api/annotation""" | |
| texts_out = [] | |
| for sent in texts: | |
| doc = nlp(" ".join(sent)) | |
| texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags]) | |
| return texts_out | |
| def process_text(texts, stop_words, bigram_mod, nlp): | |
| # Remove Stop Words | |
| data_words_nostops = remove_stopwords(texts, stop_words) | |
| # Form Bigrams | |
| data_words_bigrams = make_bigrams(data_words_nostops, bigram_mod) | |
| # Do lemmatization keeping only noun, adj, vb, adv | |
| data_lemmatized = lemmatization(data_words_bigrams, nlp, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']) | |
| return data_lemmatized | |
| def create_lda_model(corpus, id2word, num_topics=6): | |
| lda_model = gensim.models.LdaModel(corpus=corpus, | |
| id2word=id2word, | |
| num_topics=num_topics, | |
| random_state=100, | |
| chunksize=100, | |
| passes=10, | |
| per_word_topics=True) | |
| return lda_model | |
| def print_topics(lda_model): | |
| #topics = pprint(lda_model.print_topics()) | |
| topics = lda_model.print_topics() | |
| return topics | |
| def visualize_lda_model(lda_model, corpus, id2word): | |
| p = pyLDAvis.gensim.prepare(lda_model, corpus, id2word) | |
| return p | |
| def topic_modeling(reviews): | |
| # Split the input string into individual reviews | |
| responses = reviews.split("\n") | |
| # Preprocess responses | |
| preprocessed_responses = preprocess_responses(responses) | |
| # Convert responses to words | |
| data_words = list(sent_to_words(preprocessed_responses)) | |
| # Create stop words list | |
| stop_words = stopwords.words('english') | |
| stop_words.extend(['from', 'subject', 're', 'edu', 'use']) | |
| # Create bigram and trigram models | |
| bigram_mod, trigram_mod = make_bigram_trigram_models(data_words) | |
| # Initialize spacy 'en' model, keeping only tagger component (for efficiency) | |
| nlp = spacy.load("en_core_web_sm", disable=['parser', 'ner']) | |
| # Process text (remove stopwords, make bigrams, lemmatize) | |
| processed_data = process_text(data_words, stop_words, bigram_mod, nlp) | |
| # Create Dictionary and Corpus | |
| id2word = corpora.Dictionary(processed_data) | |
| corpus = [id2word.doc2bow(text) for text in processed_data] | |
| # Create LDA model | |
| lda_model = create_lda_model(corpus, id2word, num_topics=6) | |
| # Print topics | |
| topics = print_topics(lda_model) | |
| # Visualize LDA model | |
| visualization = visualize_lda_model(lda_model, corpus, id2word) | |
| visualization_html = pyLDAvis.prepared_data_to_html(visualization) | |
| current_dir = os.path.dirname(__file__) # Get the current working directory | |
| filename = f"lda_visualization.html" # Define the filename | |
| filepath = os.path.join(current_dir, filename) # Combine directory and filename | |
| with open(filepath, "w", encoding="utf-8") as file: | |
| file.write(visualization_html) # Write the HTML data to the file | |
| print("Successfully saved in", filename) | |
| return topics, visualization_html | |
| # Interface | |
| iface = gr.Interface( | |
| fn=topic_modeling, | |
| inputs="text", | |
| # outputs=["text", "html"] | |
| outputs=[ | |
| gr.Textbox(label="Topics"), | |
| gr.HTML(label="Visualization") | |
| ] | |
| ) | |
| iface.launch(share=True) | |
| #rebuild |