app.py

import streamlit as st
import pandas as pd
import numpy as np
import re
import json
import joblib
from sklearn.feature_extraction.text import TfidfVectorizer

# Impor library tambahan
#import matplotlib.pyplot as plt
#import seaborn as sns
#import plotly.express as px
from wordcloud import WordCloud
import nltk
from nltk.corpus import stopwords
#from transformers import pipeline


# Fungsi untuk membersihkan teks dengan ekspresi reguler
@st.cache_data
def clean_text(text):
    # Tahap-1: Menghapus karakter non-ASCII
    text = re.sub(r'[^\x00-\x7F]+', '', text)

    # Tahap-2: Menghapus URL
    text = re.sub(r'http[s]?://.[a-zA-Z0-9./_?=%&#+!]+', '', text)
    text = re.sub(r'pic.twitter.com?.[a-zA-Z0-9./_?=%&#+!]+', '', text)

    # Tahap-3: Menghapus mentions
    text = re.sub(r'@[\w]+', '', text)

    # Tahap-4: Menghapus hashtag
    text = re.sub(r'#([\w]+)', '', text)

    # Tahap-5 Menghapus 'amp' yang menempel pada '&' dan 'gt' yang menempel pada '&'
    text = re.sub(r'&|>', '', text)

    # Tahap-6: Menghapus karakter khusus (simbol)
    text = re.sub(r'[!$%^&*@#()_+|~=`{}\[\]%\-:";\'<>?,./]', '', text)

    # Tahap-7: Menghapus angka
    text = re.sub(r'[0-9]+', '', text)

    # Tahap-8: Menggabungkan spasi ganda menjadi satu spasi
    text = re.sub(' +', ' ', text)

    # Tahap-9: Menghapus spasi di awal dan akhir kalimat
    text = text.strip()

    # Tahap-10: Konversi teks ke huruf kecil
    text = text.lower()

    # Tahap-11: koreksi duplikasi tiga karakter beruntun atau lebih (contoh. yukkk)
    # text = re.sub(r'([a-zA-Z])\1\1', '\\1', text)
    #text = re.sub(r'(.)(\1{2,})', r'\1\1', text)
    text = re.sub(r'(\w)\1{2,}', r'\1', text)

    return text
@st.cache_data
def load_file(kamus_path, kamus_sendiri_path):
    # Membaca kamus kata gaul Salsabila  
    with open(kamus_path) as f:
        data = f.read()
    lookp_dict = json.loads(data)
    
    # Dict kata gaul saya sendiri yang tidak masuk di dict Salsabila
    with open(kamus_sendiri_path) as f:
        kamus_sendiri = f.read()
    kamus_gaul_baru = json.loads(kamus_sendiri)
    
    # Menambahkan dict kata gaul baru ke kamus yang sudah ada
    lookp_dict.update(kamus_gaul_baru)
    
    nltk.download("stopwords")
    stop_words = set(stopwords.words("indonesian"))
    return lookp_dict, stop_words
    
kamus_path = '_json_colloquial-indonesian-lexicon (1).txt'
kamus_sendiri_path = 'kamus_gaul_custom.txt'
lookp_dict, stop_words = load_file(kamus_path, kamus_sendiri_path)

# Fungsi untuk normalisasi kata gaul
@st.cache_data
def normalize_slang(text, slang_dict):
    words = text.split()
    normalized_words = [slang_dict.get(word, word) for word in words]
    return ' '.join(normalized_words)

#---------------------------------------------------NLTK Remove Stopwords----------------------------------------------------------------------

@st.cache_data
def remove_stopwords(text, stop_words):
    # Pecah teks menjadi kata-kata
    words = text.split()

    # Hapus stopwords bahasa Indonesia
    words = [word for word in words if word not in stop_words]

    return " ".join(words)
#---------------------------------------------------TFIDF----------------------------------------------------------------------  
# Memuat model TF-IDF dengan joblib (pastikan path-nya benar)
tfidf_model_path = 'X_tfidf_model.joblib'
tfidf_vectorizer = joblib.load(tfidf_model_path)
# Fungsi untuk ekstraksi fitur TF-IDF
#@st.cache_data
#def extract_tfidf_features(texts, _tfidf_vectorizer):
#    tfidf_matrix = tfidf_vectorizer.transform(texts)
#    return tfidf_matrix
#---------------------------------------------------Milih Model----------------------------------------------------------------------

# Fungsi untuk memilih model berdasarkan pilihan pengguna
@st.cache_data
def select_sentiment_model(selected_model):
    if selected_model == "Ensemble":
        model_path = 'ensemble_clf_soft_smote.joblib'
    elif selected_model == "Random Forest":
        model_path = 'best_rf_model_smote.joblib'
    elif selected_model == "Naive Bayes":
        model_path = 'naive_bayes_model_smote.joblib'
    elif selected_model == "Logistic Regression":
        model_path = 'logreg_model_smote.joblib'
    else:
        # Fallback ke model default jika pilihan tidak valid
        model_path = 'ensemble_clf_soft_smote.joblib'

    model = joblib.load(model_path)
    return model


# Fungsi untuk prediksi sentimen

def predict_sentiment(text, _model, _tfidf_vectorizer, slang_dict):
    # Tahap-1: Membersihkan dan normalisasi teks
    cleaned_text = clean_text(text)
    norm_slang_text = normalize_slang(cleaned_text, slang_dict)

    # Tahap-2: Ekstraksi fitur TF-IDF
    tfidf_matrix = _tfidf_vectorizer.transform([norm_slang_text])

    # Tahap-3: Lakukan prediksi sentimen
    sentiment = _model.predict(tfidf_matrix)

    # Tahap-4: Menggantikan indeks dengan label sentimen
    labels = {0: "Negatif", 1: "Netral", 2: "Positif"}
    sentiment_label = labels[int(sentiment)]

    return sentiment_label
@st.cache_data
def get_emoticon(sentiment):
    if sentiment == "Positif":
        emoticon = "😄"  # Emotikon untuk sentimen positif
    elif sentiment == "Negatif":
        emoticon = "😞"  # Emotikon untuk sentimen negatif
    else:
        emoticon = "😐"  # Emotikon untuk sentimen netral

    return emoticon

@st.cache_data
def buat_chart(df, target_year):
    target_year = int(target_year)
    st.write(f"Bar Chart Tahun {target_year}:")

    # Ambil bulan
    df['Date'] = pd.to_datetime(df['Date'])  # Convert 'Date' column to datetime
    df['month'] = df['Date'].dt.month
    df['year'] = df['Date'].dt.year

    # Filter DataFrame for the desired year
    df_filtered = df[df['year'] == target_year]

    # Check if data for the target year is available
    if df_filtered.empty:
        st.warning(f"Tidak ada data untuk tahun {target_year}.")
        return

    # Mapping nilai bulan ke nama bulan
    bulan_mapping = {
        1: f'Januari {target_year}',
        2: f'Februari {target_year}',
        3: f'Maret {target_year}',
        4: f'April {target_year}',
        5: f'Mei {target_year}',
        6: f'Juni {target_year}',
        7: f'Juli {target_year}',
        8: f'Agustus {target_year}',
        9: f'September {target_year}',
        10: f'Oktober {target_year}',
        11: f'November {target_year}',
        12: f'Desember {target_year}'
    }

    # Mengganti nilai dalam kolom 'month' menggunakan mapping
    df_filtered['month'] = df_filtered['month'].replace(bulan_mapping)

    # Menentukan warna untuk setiap kategori dalam kolom 'score'
    warna_label = {
        'Negatif': '#FF9AA2',
        'Netral': '#FFDAC1',
        'Positif': '#B5EAD7'
    }

    # Sorting unique scores
    unique_label = sorted(df_filtered['label'].unique())

    # Ensure months are in the correct order
    months_order = [
        f'Januari {target_year}', f'Februari {target_year}', f'Maret {target_year}', f'April {target_year}', f'Mei {target_year}', f'Juni {target_year}',
        f'Juli {target_year}', f'Agustus {target_year}', f'September {target_year}', f'Oktober {target_year}', f'November {target_year}', f'Desember {target_year}'
    ]

    # Sort DataFrame based on the custom order of months
    df_filtered['month'] = pd.Categorical(df_filtered['month'], categories=months_order, ordered=True)
    df_filtered = df_filtered.sort_values('month')

    # Create a bar chart with stacking and manual colors
    st.bar_chart(
        df_filtered.groupby(['month', 'label']).size().unstack().fillna(0),
        color=[warna_label[label] for label in unique_label]
    )
    
# Fungsi untuk membuat tautan unduhan
@st.cache_data
def get_table_download_link(df, download_format):
    if download_format == "XLSX":
        df.to_excel("hasil_sentimen.xlsx", index=False)
        return f'<a href="hasil_sentimen.xlsx" download="hasil_sentimen.xlsx">Unduh File XLSX</a>'
    else:
        csv = df.to_csv(index=False)
        return f'<a href="data:file/csv;base64,{b64encode(csv.encode()).decode()}" download="hasil_sentimen.csv">Unduh File CSV</a>'


# Judul
st.title("Analisis Sentimen Based on Tweets Biskita Transpakuan")
preference_barchart_date = False
#-----------------------------------------------------General Settings---------------------------------------------------------------
with st.expander("General Settings :"):
    # Tambahkan widget untuk memilih model
    selected_model = st.selectbox("Pilih Model Sentimen:", ("Ensemble", "Naive Bayes", "Logistic Regression", "Transformer"))
    
    # Memilih model sentimen berdasarkan pilihan pengguna
    sentiment_model = select_sentiment_model(selected_model)
    
    # Pilihan input teks manual atau berkas XLSX
    input_option = st.radio("Pilih metode input:", ("Teks Manual", "Unggah Berkas XLSX"))
    
    if input_option == "Teks Manual":
        # Input teks dari pengguna
        user_input = st.text_area("Masukkan teks:", "")
    else:
        # Input berkas XLSX
        uploaded_file = st.file_uploader("Unggah berkas XLSX", type=["xlsx"])
        st.info("Pastikan berkas XLSX Anda memiliki kolom yang bernama 'Text'.")
    
        if uploaded_file is not None:
            df = pd.read_excel(uploaded_file)
    
            if 'Text' not in df.columns:
                st.warning("Berkas XLSX harus memiliki kolom bernama 'Text' untuk analisis sentimen.")
                if not df['Text'].empty:
                    st.warning("Kolom 'Text' harus mempunyai value.")
            else:
                texts = df['Text']  # Sesuaikan dengan nama kolom di berkas XLSX Anda
            if "Date" in df.columns :
                if not df['Date'].empty:
                    dates = df['Date']
                    preference_barchart_date = True
                    
#-----------------------------------------------------Preference Settings--------------------------------------------------
with st.expander ("Preference Settings :"):
    colormap = st.selectbox("Pilih Warna Wordclouds :", ["Greys", "Purples", "Blues", "Greens", "Oranges", "Reds", "YlOrBr", "YlOrRd", "OrRd", "PuRd", "RdPu", "BuPu", "GnBu", "PuBu", "YlGnBu", "PuBuGn", "BuGn", "YlGn"])
    if preference_barchart_date == True:
        bar = st.selectbox("Pilih Tampilan Bar Chart :", ("Distribusi Kelas", "Distribusi Kelas Berdasarkan Waktu"), index = 0)
        target_year = st.selectbox("Pilih Tahun Bar Chart :", df['Date'].str[:4].unique())
# Analisis sentimen
results = []
analisis = False
if input_option == "Teks Manual" and user_input:
    if st.button("Analysis"):
        # Pisahkan teks yang dimasukkan pengguna menjadi baris-baris terpisah
        user_texts = user_input.split('\n')
        for text in user_texts:
            sentiment_label = predict_sentiment(text, sentiment_model, tfidf_vectorizer, lookp_dict)
            emoticon = get_emoticon(sentiment_label)
            cleaned_text = clean_text(text)
            norm_slang_text = normalize_slang(cleaned_text, lookp_dict)
            tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words)
            
            results.append({
                'Text': text, 
                'cleaned-text' : cleaned_text, 
                'normalisasi-text' : norm_slang_text, 
                'stopwords-remove' : tanpa_stopwords,
                'label' : sentiment_label,
                'emotikon' : emoticon,
            })
            analisis = True

elif input_option == "Unggah Berkas XLSX" and uploaded_file is not None:
    if st.button("Analysis"):
        if 'Text' in df.columns:
            if 'Date' in df.columns:
                for text, date in zip(texts, df['Date']):
                    sentiment_label = predict_sentiment(text, sentiment_model, tfidf_vectorizer, lookp_dict)
                    emoticon = get_emoticon(sentiment_label)
                    cleaned_text = clean_text(text)
                    norm_slang_text = normalize_slang(cleaned_text, lookp_dict)
                    tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words)
        
                    result_entry = {
                        'Text': text,
                        'cleaned-text': cleaned_text,
                        'normalisasi-text': norm_slang_text,
                        'stopwords-remove': tanpa_stopwords,
                        'label': sentiment_label,
                        'emotikon': emoticon,
                        'Date': date,
                    }
        
                    results.append(result_entry)
        
                analisis = True
            else:
                for text in texts:
                    sentiment_label = predict_sentiment(text, sentiment_model, tfidf_vectorizer, lookp_dict)
                    emoticon = get_emoticon(sentiment_label)
                    cleaned_text = clean_text(text)
                    norm_slang_text = normalize_slang(cleaned_text, lookp_dict)
                    tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words)
        
                    result_entry = {
                        'Text': text,
                        'cleaned-text': cleaned_text,
                        'normalisasi-text': norm_slang_text,
                        'stopwords-remove': tanpa_stopwords,
                        'label': sentiment_label,
                        'emotikon': emoticon,
                    }
        
                    results.append(result_entry)
        
                analisis = True   
        else:
            st.warning("Berkas XLSX harus memiliki kolom bernama 'Text' untuk analisis sentimen.")

        
st.info('Tekan "Analysis" kembali jika tampilan menghilang', icon = 'ℹ️')
if results and analisis == True:
    df_results = pd.DataFrame(results)
    # Membagi tampilan menjadi dua kolom
    columns = st.columns(2)
    
    # Kolom pertama untuk Word Cloud
    with columns[0]:
        st.write("Wordclouds :")
        all_texts = [result['stopwords-remove'] for result in results if result['stopwords-remove'] is not None and not pd.isna(result['stopwords-remove'])]
        all_texts = " ".join(all_texts)
    
        if all_texts:
            wordcloud = WordCloud(width=800, height=660, background_color='white',
                                  colormap=colormap,   # Warna huruf
                                  contour_color='black',  # Warna kontur
                                  contour_width=2,       # Lebar kontur
                                  mask=None,             # Gunakan mask untuk bentuk kustom
                                  ).generate(all_texts)
            st.image(wordcloud.to_array())
        else:
            st.write("Tidak ada data untuk ditampilkan dalam Word Cloud.")

    if 'Date' in df_results.columns:
        if bar == "Distribusi Kelas Berdasarkan Waktu":
            if not df_results['Date'].empty:
                with columns[1]:
                    buat_chart(df_results, target_year)
        else :
            # Kolom kedua untuk Bar Chart
            with columns[1]:
                st.write("Bar Chart :")   
                # Membuat bar chart
                st.bar_chart(
                    df_results["label"].value_counts()
                )
    else :
        # Kolom kedua untuk Bar Chart
        with columns[1]:
            st.write("Bar Chart :")   
            # Membuat bar chart
            st.bar_chart(
                df_results["label"].value_counts()
            )
    # Menampilkan hasil analisis sentimen dalam kotak yang dapat diperluas
    with st.expander("Hasil Analisis Sentimen"):
        # Tampilkan tabel hasil analisis sentimen
        st.write(pd.DataFrame(results))
    
    if results:
        # Simpan DataFrame ke dalam file CSV
        df = pd.DataFrame(results)
        csv = df.to_csv(index=False)
    
        # Tampilkan tombol unduh CSV
        st.download_button(label="Unduh CSV", data=csv, key="csv_download", file_name="hasil_sentimen.csv")
    else:
        st.write("Tidak ada data untuk diunduh.")


# Garis pemisah
st.divider()

# Tautan ke GitHub
github_link = "https://github.com/naufalnashif/"
st.markdown(f"GitHub: [{github_link}]({github_link})")

# Tautan ke Instagram
instagram_link = "https://www.instagram.com/naufal.nashif/"
st.markdown(f"Instagram: [{instagram_link}]({instagram_link})")

# Pesan penutup
st.write('Thank you for trying the demo!')
st.write('Best regards, Naufal Nashif')