from transformers import pipeline import matplotlib.pyplot as plt import streamlit as st import langid import pandas as pd from difflib import SequenceMatcher import random def calculate_similarity(a, b): return SequenceMatcher(None, a, b).ratio() def filter_similar_items(list, similarity_threshold): filtered_data = [] for item in list: is_similar = False for saved_item in filtered_data: similarity = calculate_similarity(item, saved_item) if similarity > similarity_threshold: is_similar = True break if not is_similar: filtered_data.append(item) return filtered_data def process_data(input_data,columnname = 'text', num_data = 100): random.seed(20979738) processed_data = [i for i in input_data[columnname]] random_selection = random.sample(processed_data, num_data) filtered_data = filter_similar_items(random_selection, similarity_threshold = 0.75) st.write('Number of data input: ',len(random_selection)) st.write('After removing duplicates: ',len(filtered_data)) return filtered_data def chi2eng(filtered_data): translated_data = [] language_Classification = langid.classify(filtered_data[0])[0] if language_Classification == "zh": st.write("Your input is Chinese, translating to English") st.write('▶️ Loading Translation Model...') trans_pipe = pipeline("translation", model="Helsinki-NLP/opus-mt-zh-en") st.write('⏺️ Translation Model Successfully Loaded') st.write('▶️ Start Translating...') translation_progress_count = 0 translation_bar = st.progress(0) for i in filtered_data: translated_data.append(trans_pipe(i)[0]['translation_text']) translation_progress_count += 1/len(filtered_data) translation_bar.progress(translation_progress_count) elif language_Classification == 'en': st.write("Your input is English, moving to next stage...") translated_data = [i for i in filtered_data] else: st.write('The anguage you input is: ',langid.classify(filtered_data[0])[0],'the program cannot process') return translated_data # Text Classification：Negative/Neutral/Positive def emotion_classification(translated_data): st.write('▶️ Loading Classification Model...') emo_pipe = pipeline("text-classification", model="deeplearningwithpython5240/twitter_roberta_base_sentiment_fintune_with_app_reviews") st.write('⏺️ Classification Model Successfully Loaded') negative_count, neutral_count, positive_count = 0,0,0 negative_dict = {} emotion_progress_count = 0 st.write('▶️ Data Processing ...') emotion_bar = st.progress(0) for i in translated_data: labelled_result = emo_pipe(i)[0]['label'] # st.write('Text: ',i) # st.write('Label: ',labelled_result) # st.write(' ') if labelled_result == 'negative': negative_dict[i] = emo_pipe(i)[0]['score'] negative_count += 1 if labelled_result == 'neutral': neutral_count += 1 if labelled_result == 'positive': positive_count += 1 emotion_progress_count += 1/len(translated_data) emotion_bar.progress(emotion_progress_count) sizes = [negative_count, neutral_count, positive_count] labels = ['Negative_Reviews', 'Neutral_Reviews', 'Positive_Reviews'] # 创建饼状图 st.write('Number of Positive Reviews: ', positive_count) st.write('Number of Neutral Reviews: ', neutral_count) st.write('Number of Negative Reviews: ', negative_count) plt.figure(figsize=(5, 5)) # 设置图表大小 plt.pie(sizes, labels=labels, autopct='%1.1f%%', startangle=90) # 显示图表 st.pyplot(plt.show()) negative_dict_sorted = dict(sorted(negative_dict.items(), key=lambda x: x[1], reverse=True)) top10_negative_str = "" if len(negative_dict_sorted) < 10: st.write("Total Number of Negative Comments: ",len(negative_dict_sorted)) for k,v in negative_dict_sorted.items(): st.write(k) top10_negative_str += f"{k}." else: st.write("Top 10 Negative Comments") count = 0 for k,v in negative_dict_sorted.items(): if count >= 10: break st.write(k) top10_negative_str += f"{k}." count += 1 return top10_negative_str # Summarization def summarization(top10_negative_str): st.write('▶️ Loading Summarizatio Model...') summarize_pipe = pipeline("text2text-generation", model="deeplearningwithpython5240/summarisation-t5-finetuned-model", max_new_tokens =512) st.write('⏺️ Summarization Model Successfully Loaded') st.write('▶️ Summarizing...') summarized_text = summarize_pipe(top10_negative_str) return summarized_text def main(): st.set_option('deprecation.showPyplotGlobalUse', False) st.set_page_config(page_title="Review Sentiment Analysis and Improvement Summarisation Report for Business Product", page_icon="🦜") st.header("Review Sentiment Analysis and Improvement Summarisation Report for Business Product") st.write(f"Welcome to the user guide for our product feedback analysis application. Our application is designed to help companies review their product feedback and summarise areas for improvement. Here are the steps to get started:\n1. Upload reviews in CSV file format\n2. Input column name and number of data to be analysed\n3. Automatically removes duplicate reviews\n4. Translate data into English (if the dataset is Chinese)\n5. Analyse reviews sentiment\n6. Summarize top negative reveiws") try: uploaded_file = st.file_uploader("🔶 Upload CSV file for analysis 🔶", type={"csv"}) if uploaded_file is not None: columnname = st.text_input("🔶 Please enter the column name in CSV file you want to analyze 🔶") num_data = st.number_input("🔶 Please enter the number of rows you want to process 🔶",min_value=2, step=1) input_data = pd.read_csv(uploaded_file) st.dataframe(input_data) st.text('️️ ') #stage 1：process data st.text('🔶 Processing Data 🔶') processed_data = process_data(input_data ,columnname, int(num_data)) st.write(processed_data) st.text('️️🟢 Processing Data Finished 🟢') st.text('️️ ') #stage 2：translate st.text('🔶 Checking Translation is Needed or Not 🔶') translated_data = chi2eng(processed_data) st.write(translated_data) st.text('️️🟢 Translation Finished 🟢') st.text('️️ ') #stage 3：emotion Classification st.text('️️🔶 Processing Emotion Classification 🔶') top10_negative_str = emotion_classification(translated_data) st.text('️️🟢 Emotion Classification Finished 🟢') st.text('️️ ') #stage 4：Summarization st.text('🔶 Processing Summarization 🔶') if len(top10_negative_str) == 0: st.write("No Negative Reviews Detected") else: summarized_text = summarization(top10_negative_str) st.write('Summarized Negative Comments:') st.write(summarized_text[0]["generated_text"]) st.text('️️🟢 Summarization Finished 🟢') except: st.write("") if __name__ == "__main__": main()