import gradio as gr from transformers import pipeline, AutoTokenizer from classifier import MistralForSequenceClassification import torch import nltk import json import pandas as pd import plotly.graph_objects as go from wordcloud import WordCloud import matplotlib.pyplot as plt import io import base64 from PIL import Image from nltk import bigrams import malaya from collections import Counter HF_TOKEN = os.getenv('hf_token') hf_writer = gr.HuggingFaceDatasetSaver(HF_TOKEN,'aisyahhrazak/tpb-crowdsourced-dataset') with open('en.json') as fopen: en = json.load(fopen) stopwords = malaya.text.function.get_stopwords() stopwords = stopwords + en + ['lor', 'quote','Quote','QUOTE','pm', 'long', 'jer', 'time', 'feel', 'liao', 'wow', 'https', 'http', 've', 'ko', 'kena', 'post', 'ni', 'tu', 'don', 'je', 'jeh', 'la', 'tau', 'haha', 'hahaha', 'hahahaha'] stopwords += ['for me', 'to be', 'in the', 'me to', 'for me to'] nltk.download('punkt', quiet=True) nltk.download('punkt_tab', quiet=True) nltk.download('stopwords', quiet=True) nltk.download('vader_lexicon', quiet=True) tokenizer_tpb = AutoTokenizer.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512') model_tpb = MistralForSequenceClassification.from_pretrained('HalalFoodNLP/tpb-model-halal', torch_dtype=torch.bfloat16) model_sentiment = MistralForSequenceClassification.from_pretrained('malaysia-ai/sentiment-mistral-191M-MLM', torch_dtype=torch.bfloat16) pipeline_tpb = pipeline(task="text-classification", model=model_tpb, tokenizer=tokenizer_tpb) sentiment_pipeline = pipeline("sentiment-analysis", model=model_sentiment, tokenizer=tokenizer_tpb) data = [] with open('sentiment-tpb-dataset.jsonl', 'r') as file: for line in file: data.append(json.loads(line)) df = pd.DataFrame(data) # Update the generate_wordcloud function to return a PIL Image object def generate_wordcloud(text): # Generate the word cloud wordcloud = WordCloud(width=300, height=200, background_color='white').generate(text) # Create the plot plt.figure(figsize=(10, 5)) plt.imshow(wordcloud, interpolation='bilinear') plt.axis('off') plt.tight_layout(pad=0) # Save the plot to a bytes buffer buf = io.BytesIO() plt.savefig(buf, format='png') plt.close() buf.seek(0) # Convert bytes buffer to PIL Image image = Image.open(buf) return image # Add a function to generate bigrams def generate_bigrams(text): words = nltk.word_tokenize(text.lower()) words = [word for word in words if word.isalnum() and word not in stopwords] bi_grams = list(bigrams(words)) return Counter(bi_grams).most_common(10) def predict_decision(sentiment_label): if sentiment_label == 'positive': return "High likelihood of purchase" elif sentiment_label == 'neutral': return "Moderate likelihood of purchase" else: return "Low likelihood of purchase" # Function to generate report based on TPB sentiment def generate_report(tpb_sentiment_df): report = "## TPB Factor Analysis and Recommendations Report\n\n" for _, row in tpb_sentiment_df.iterrows(): tpb_label = row['tpb_label'] positive_percentage = row['positive'] negative_percentage = row['negative'] if negative_percentage > 70: # Only generate recommendations for positive < 70% if tpb_label == "attitude": report += f"### {tpb_label.capitalize()} ({negative_percentage:.1f}% Negative)\n" report += """ **Current Issues:** - High negative perception regarding product quality - Concerns about halal certification and its authenticity - Pricing issues in comparison to perceived value **Recommended Actions:** 1. **Quality Control Improvements** - Implement enhanced product quality measures - Obtain globally recognized halal certifications - Conduct regular quality audits 2. **Educational Campaigns** - Educate customers on halal certification processes - Raise awareness about the health benefits of halal products - Highlight ethical and sustainable sourcing 3. **Pricing Strategy Adjustment** - Reassess pricing to align with customer expectations - Introduce discount programs or loyalty initiatives """ if tpb_label == "religious knowledge": report += f"### {tpb_label.capitalize()} ({negative_percentage:.1f}% Negative)\n" report += """ **Current Issues:** - Lack of awareness and understanding about the halal process - Customers may be unsure of the religious guidelines followed **Recommended Actions:** 1. **Religious Knowledge Enhancement** - Provide clear educational materials on the halal process - Collaborate with religious scholars to endorse products - Ensure transparent labeling and certification 2. **Community Engagement** - Host webinars or community events about halal - Partner with local religious organizations for outreach - Share customer testimonials emphasizing trust in your certification """ if tpb_label == "subjective norms": report += f"### {tpb_label.capitalize()} ({negative_percentage:.1f}% Negative)\n" report += """ **Current Issues:** - Social influence or peer pressure regarding halal compliance is weak - Lack of community-driven recommendations for the product **Recommended Actions:** 1. **Influence Social Circles** - Engage community leaders or influencers to endorse products - Create social campaigns around the halal certification to enhance peer recommendations 2. **Referral Programs** - Introduce referral programs where existing customers can promote the product - Offer incentives for customers who share their experiences with others 3. **Testimonials and Success Stories** - Use customer testimonials and success stories to strengthen social trust """ if tpb_label == "perceived behavioural control": report += f"### {tpb_label.capitalize()} ({negative_percentage:.1f}% Negative)\n" report += """ **Current Issues:** - Perceived difficulty in understanding or accessing halal-certified products - Concerns about control over product quality and sourcing transparency **Recommended Actions:** 1. **Improve Accessibility** - Make halal products more accessible through multiple platforms (e-commerce, retail stores) - Ensure ease of purchase and fast delivery options 2. **Enhance Transparency** - Provide detailed information about sourcing and production processes - Use blockchain or similar technology to enhance transparency in halal certification 3. **Customer Empowerment** - Offer customer feedback channels to empower users to voice concerns and suggestions - Ensure that concerns are addressed promptly to build trust and satisfaction """ return report def search_company(keyword): if not keyword: return None, None, None, None filtered_df = df[df['text'].str.contains(keyword, case=False)] if filtered_df.empty: return None, None, None, None # Calculate sentiment distribution sentiment_counts = filtered_df['label'].value_counts(normalize=True) * 100 colors = ['red' if sentiment == 'negative' else 'gray' if sentiment == 'neutral' else 'blue' for sentiment in sentiment_counts.index] # Create the bar plot sentiment_fig = go.Figure(data=[go.Bar( x=sentiment_counts.index, y=sentiment_counts.values, text=[f'{val:.1f}%' for val in sentiment_counts.values], textposition='auto', marker_color=colors )]) sentiment_fig.update_layout( title='Overall Sentiment Distribution', xaxis_title='Sentiment', yaxis_title='Percentage' ) tpb_counts = filtered_df['tpb_label'].value_counts(normalize=True) * 100 tpb_fig = go.Figure(data=[go.Bar( x=tpb_counts.index, y=tpb_counts.values, text=[f'{val:.1f}%' for val in tpb_counts.values], textposition='auto' )]) tpb_fig.update_layout(title='Overall TPB Factor Distribution', xaxis_title='TPB Factor', yaxis_title='Percentage') # Calculate sentiment distribution within each TPB factor tpb_sentiment_df = filtered_df.groupby(['tpb_label', 'label']).size().unstack(fill_value=0) tpb_sentiment_df = tpb_sentiment_df.div(tpb_sentiment_df.sum(axis=1), axis=0) * 100 # Define colors for each sentiment color_map = { 'negative': 'red', 'neutral': 'gray', 'positive': 'blue' } tpb_sentiment_fig = go.Figure() for sentiment in tpb_sentiment_df.columns: tpb_sentiment_fig.add_trace(go.Bar( name=sentiment, x=tpb_sentiment_df.index, y=tpb_sentiment_df[sentiment], text=[f'{val:.1f}%' for val in tpb_sentiment_df[sentiment]], textposition='auto', marker_color=color_map.get(sentiment, 'gray') )) tpb_sentiment_fig.update_layout( barmode='stack', title='Sentiment Distribution within TPB Factors', xaxis_title='TPB Factor', yaxis_title='Percentage' ) report = generate_report(tpb_sentiment_df.reset_index()) wordclouds = {} bigrams_data = {} for label in filtered_df['tpb_label'].unique(): text = ' '.join(filtered_df[filtered_df['tpb_label'] == label]['text']).replace('QUOTE','').replace('quote','').replace('sijil halal','').replace('halal','') wordclouds[label] = generate_wordcloud(text) bigrams_data[label] = generate_bigrams(text) # Extract only the words words_only = { key: [word_pair for word_pair, _ in value] for key, value in bigrams_data.items() } # Create a single DataFrame for bigrams, with only the bigram text (no frequency) bigram_df = pd.DataFrame({ label: data for label, data in words_only.items() }) print(bigrams_data.items()) bigram_df.index = [f"Top {i+1}" for i in range(len(bigram_df))] return (sentiment_fig, tpb_fig, tpb_sentiment_fig, filtered_df[filtered_df['text'].str.len() < 300].head(5), report, wordclouds.get('attitude'), wordclouds.get('religious knowledge'), wordclouds.get('subjective norms'), wordclouds.get('perceived behavioural control'),bigram_df) def text_classification_and_sentiment(text, keywords_df): result_tpb = pipeline_tpb(text) tpb_label = result_tpb[0]['label'] tpb_score = result_tpb[0]['score'] result_sentiment = sentiment_pipeline(text) sentiment_label = result_sentiment[0]['label'] sentiment_score = result_sentiment[0]['score'] keywords_df = pd.read_excel('IMG_8137.xlsx') # Check for keywords in the first column of the DataFrame keywords = keywords_df.iloc[:, 0].tolist() for keyword in keywords: if keyword.lower() in text.lower(): sentiment_label = 'negative' sentiment_score = 1.0 decision = predict_decision(sentiment_label) tpb_output = f"TPB Label: {tpb_label}" sentiment_output = f"Sentiment: {sentiment_label}\nProbability: {sentiment_score * 100:.2f}%" decision_output = f"Decision: {decision}" return tpb_output, sentiment_output, decision_output examples = [ "Alhamdulillah, hari ni dapat makan dekat restoran halal baru. Rasa puas hati dan tenang bila tau makanan yang kita makan dijamin halal.", "Semua orang cakap kena check logo halal sebelum beli makanan. Dah jadi macam second nature dah sekarang. Korang pun sama kan?" ] css = """ :root { --bg: #FFFFFF; /* Set the background color to white */ --col: #191919; /* Define primary text color */ --bg-dark: #000000; /* Define dark background color if needed */ --col-dark: #ECF2F7; /* Define dark text color if needed */ ----body-background-fill: #FFFFFF; } html, body { background-color: var(--bg); /* Set the background color to white for the entire page */ margin: 0; /* Remove default body margin */ padding: 0; /* Remove default body padding */ } .container { max-width: 1000px; margin: auto; padding: 20px; } .title { text-align: center; margin-bottom: 20px; } .nav-buttons { display: flex; justify-content: center; gap: 10px; margin-bottom: 20px; } #recommendation_report { background-color: #f9f9f9; /* Keep this background light for the report section */ padding: 20px; border: 2px solid #e0e0e0; border-radius: 10px; margin-top: 20px; font-family: Arial, sans-serif; font-size: 14px; } .wrap-text { white-space: normal !important; word-wrap: break-word; } .footer {visibility: hidden} """ with gr.Blocks(css=css + """ body, .gradio-container, .root, .wrap, #root .background .container { background-color: white !important; background-image: none !important; background-fill: white !important; } """, theme='aisyahhrazak/miku-aisyah@=1.2.2') as demo: with gr.Tabs() as tabs: with gr.TabItem("User View", id=0): gr.Markdown("## Text Classification and Sentiment Analysis Based on User Input About Halal Food Acquisition") gr.Markdown("Enter a text to see TPB classification, sentiment analysis, and purchase prediction results!") input_text = gr.Textbox(lines=2, label="Input Comment", placeholder="Model can make mistakes, we are striving to improve the model.") with gr.Row(): tpb_output = gr.Textbox(lines=3, label="TPB Classification") sentiment_output = gr.Textbox(lines=3, label="Sentiment Analysis") decision_output = gr.Textbox(lines=3, label="Purchase Prediction") # This needs to be called at some point prior to the first call to callback.flag() hf_writer.setup([input_text,tpb_output, sentiment_output], "flagged_data_points") classify_button = gr.Button("Analyze") classify_button.click(lambda *args: hf_writer.flag(list(args)),fn=text_classification_and_sentiment, inputs=input_text, outputs=[tpb_output, sentiment_output, decision_output]) gr.Examples(examples=examples, inputs=input_text) with gr.TabItem("Company View", id=1): gr.Markdown("# Sentiment Analysis and Purchase Decision Factor for Halal Food Acquisition") input_text = gr.Textbox(lines=1, label="Search Keyword", placeholder="Enter keyword") search_button = gr.Button("Search") with gr.Row(): sentiment_chart = gr.Plot(label="Sentiment Distribution") tpb_chart = gr.Plot(label="TPB Factor Distribution") tpb_sentiment_chart = gr.Plot(label="Sentiment Distribution within TPB Factors") # Update word cloud outputs to be in a single row gr.Markdown("### Word Clouds by TPB Label") with gr.Row(): attitude_wc = gr.Image(label="Attitude Word Cloud", height=200, width=300) religious_knowledge_wc = gr.Image(label="Religious Knowledge Word Cloud", height=200, width=300) subjective_norms_wc = gr.Image(label="Subjective Norms Word Cloud",height=200, width=300) perceived_behavioural_control_wc = gr.Image(label="Perceived Behavioural Control Word Cloud", height=200, width=300) with gr.Accordion("See Recommendation Details"): report_output = gr.Markdown(label="Recommendation Report", elem_id="recommendation_report") gr.Markdown("### Top Bigrams by TPB Label") bigram_table = gr.Dataframe(label="Top Bigrams for Each TPB Label") output_table = gr.Dataframe( headers=["text", "tpb_label", "sentiment", "score"], label="Company Analysis Results", wrap=True ) search_button.click( fn=search_company, inputs=input_text, outputs=[ sentiment_chart, tpb_chart, tpb_sentiment_chart, output_table, report_output, attitude_wc, religious_knowledge_wc, subjective_norms_wc, perceived_behavioural_control_wc,bigram_table ] ) demo.launch()