app.py

import streamlit as st
import pandas as pd
#from google_play_scraper import app, Sort, reviews, permission, reviews_all, search
from google_play_scraper import app, Sort, reviews, reviews_all, permissions, search
import re
from datetime import datetime
import pytz

#---------------------------------------------func----------------------------------

@st.cache_data
def get_url_by_app_name(nama_apl):
    """
    Mengembalikan URL aplikasi berdasarkan nama aplikasi dari kamus.

    Parameters:
    - nama_apl (str): Nama aplikasi yang dicari.
    - aplikasi_dict (dict): Kamus yang memetakan nama aplikasi ke URL.

    Returns:
    - str or None: URL aplikasi atau None jika tidak ditemukan.
    """
    list_url = [
        'https://play.google.com/store/apps/details?id=com.shopee.id',
        'https://play.google.com/store/apps/details?id=com.tokopedia.tkpd',
        'https://play.google.com/store/apps/details?id=com.amazon.mShop.android.shopping',
        'https://play.google.com/store/apps/details?id=com.grabtaxi.passenger'
    ]
    aplikasi_dict = {
        'Shopee': list_url[0],
        'Tokopedia': list_url[1],
        'Amazon': list_url[2],
        'Grab': list_url[3]
    }
    return aplikasi_dict.get(nama_apl, None)
    
@st.cache_data
def extract_app_id(play_store_url):
    # Definisikan pola ekspresi reguler untuk menemukan ID aplikasi
    pattern = r'id=([a-zA-Z0-9._]+)'

    # Gunakan ekspresi reguler untuk mencocokkan pola dalam URL
    match = re.search(pattern, play_store_url)

    # Periksa apakah ada kecocokan dan kembalikan ID aplikasi jika ada
    if match:
        app_id = match.group(1)
        return app_id
    else:
        return None
@st.cache_data(show_spinner = 'On progress, please wait...')
def scraping_func(app_id, bahasa, negara, filter_score, jumlah):
    filter_score = None if filter_score == "Semua Rating" else filter_score
    
    rws, token = reviews(   
        app_id,
        lang=bahasa,
        country=negara,
        sort=Sort.NEWEST,
        filter_score_with=filter_score,
        count=jumlah
    )
    
    scraping_done = bool(rws)
    
    return rws, token, scraping_done

@st.cache_data(show_spinner = 'On progress, please wait...')
def scraping_all_func(app_id, bahasa, negara, filter_score, sleep = 0):
    filter_score = None if filter_score == "Semua Rating" else filter_score
    
    rws = reviews_all(   
        app_id,
        sleep_milliseconds=sleep, # defaults to 0
        lang=bahasa,
        country=negara,
        filter_score_with=filter_score,
    )
    
    scraping_done = bool(rws)
    
    return rws, scraping_done

@st.cache_data
def buat_chart(df, target_year):
    st.write(f"Bar Chart Tahun {target_year}:")

    # Ambil bulan
    df['at'] = pd.to_datetime(df['at'])  # Convert 'at' column to datetime
    df['month'] = df['at'].dt.month
    df['year'] = df['at'].dt.year

    # Filter DataFrame for the desired year
    df_filtered = df[df['year'] == target_year]

    # Check if data for the target year is available
    if df_filtered.empty:
        st.warning(f"Tidak ada data untuk tahun {target_year}.")
        return

    # Mapping nilai bulan ke nama bulan
    bulan_mapping = {
        1: f'Januari {target_year}',
        2: f'Februari {target_year}',
        3: f'Maret {target_year}',
        4: f'April {target_year}',
        5: f'Mei {target_year}',
        6: f'Juni {target_year}',
        7: f'Juli {target_year}',
        8: f'Agustus {target_year}',
        9: f'September {target_year}',
        10: f'Oktober {target_year}',
        11: f'November {target_year}',
        12: f'Desember {target_year}'
    }

    # Mengganti nilai dalam kolom 'month' menggunakan mapping
    df_filtered['month'] = df_filtered['month'].replace(bulan_mapping)

    # Menentukan warna untuk setiap kategori dalam kolom 'score'
    warna_score = {
        1: '#FF9AA2',
        2: '#FFB7B2',
        3: '#FFDAC1',
        4: '#E2F0CB',
        5: '#B5EAD7'
    }

    # Sorting unique scores
    unique_scores = sorted(df_filtered['score'].unique())

    # Ensure months are in the correct order
    months_order = [
        f'Januari {target_year}', f'Februari {target_year}', f'Maret {target_year}', f'April {target_year}', f'Mei {target_year}', f'Juni {target_year}',
        f'Juli {target_year}', f'Agustus {target_year}', f'September {target_year}', f'Oktober {target_year}', f'November {target_year}', f'Desember {target_year}'
    ]

    # Sort DataFrame based on the custom order of months
    df_filtered['month'] = pd.Categorical(df_filtered['month'], categories=months_order, ordered=True)
    df_filtered = df_filtered.sort_values('month')

    # Create a bar chart with stacking and manual colors
    st.bar_chart(
        df_filtered.groupby(['month', 'score']).size().unstack().fillna(0),
        color=[warna_score[score] for score in unique_scores]
    )

utc_timezone = pytz.timezone('UTC')
datetime_utc = datetime.now(utc_timezone)
wib_timezone = pytz.timezone('Asia/Jakarta')
dateNow = datetime_utc.astimezone(wib_timezone)

# dateNow = datetime.now(timezone.utc)
dateSimple = dateNow.strftime("%A, %d %b %Y")
timeNow = dateNow.strftime("%H:%M WIB")
yearNow = dateNow.strftime("%Y")


#--------------------------------------------UI---------------------------------------
# Streamlit UI
st.title("Data Everywhere : Scraping Playstore Reviews")
scraping_done = False
with st.sidebar :
    st.text(f"Today\t: {dateSimple}")
    st.text(f"Time\t: {timeNow}")
    with st.expander("Scraping Settings :"):
        scrape = st.selectbox("PIlih Metode :", ("Semua Reviews", "Estimasi Data"), index = 1)
        aplikasi = st.radio( 
            "Pilih Input :",
            ["Defaults", "Custom URL"], index = 0,
            captions = ["Shopee, Tokopedia, Amazon, Grab", "Tambahkan URL Manual"])
        if aplikasi == "Defaults" :
            nama_apl = st.selectbox("Pilih Aplikasi :", ('Shopee', 'Tokopedia', 'Amazon', 'Grab'))
            if nama_apl :
                url = get_url_by_app_name(nama_apl)
        elif aplikasi == "Custom URL":
            url = st.text_input("Masukkan URL Aplikasi Pada Web Playstore :", 'https://play.google.com/store/apps/details?id=com.shopee.id')
        if scrape == "Estimasi Data" :
            jumlah = st.number_input("Masukkan Estimasi Banyak Data :", min_value = 10, max_value = 25000, step = 10, placeholder="Type a number...")
    with st.expander("Preference Settings :"):
        if scrape == "Semua Reviews" : 
            sleep = st.number_input("Masukkan sleep (milisecond) :", min_value = 1, max_value = 1000, step = 10, placeholder="Type a number...")
        bahasa = st.selectbox("Pilih Bahasa:", ('en', 'id'))
        negara = st.selectbox("Pilih Negara :", ('us', 'id'))
        filter_score = st.selectbox("Pilih Rating :", ('Semua Rating', 1, 2, 3, 4, 5))
        target_year = st.selectbox("Pilih Tahun Bar Chart :", (2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025), index = 7)
        download_format = st.selectbox("Pilih Format Unduhan :", ["XLSX", "CSV", "JSON"])
    st.info('Tekan "Mulai Scraping" kembali jika tampilan menghilang ', icon="ℹ️")
    
    if url and bahasa and negara and filter_score and download_format:
        if st.button ("Mulai Scraping") :
            app_id = extract_app_id(url)
            if scrape == "Semua Reviews" :
                reviews, scraping_done = scraping_all_func(app_id, bahasa, negara, filter_score, sleep)
                df = pd.DataFrame(reviews)
            elif scrape == "Estimasi Data":
                reviews, token, scraping_done = scraping_func(app_id, bahasa, negara, filter_score, jumlah)
                df = pd.DataFrame(reviews)
            else :
                st.warning("Masukkan pilihan yang valid")
    else :
        st.error("Mohon Masukkan Parameter.")

tab1, tab2, tab3, tab4 = st.tabs(["📋 User Guide", "📈 Results", "🤵 Creator", "🔍 More"])
with tab1:
    @st.cache_resource
    def tab_1():
        st.header("User Guide:")
        '''
        Langkah - langkah :
        1. Buka sidebar sebelah kiri
        2. Buka Scraping Settings
        3. Hati - hati jika menggunakan "Semua Reviews" karena bisa berjumlah jutaan data
        4. Masukkan URL app pada situs playstore
        5. Sesuaikan bahasa, negara, dan rating yang akan diambil
        6. Pilih tahun bar chart
        7. Pilih format unduhan
        8. Klik "Mulai Scraping"
        9. Buka tab Results
        '''
    tab_1()
#-------------------------------------------BE----------------------------------------

with tab2:
    st.header("Results:")

    if scraping_done == True:
        with st.expander(f"Hasil Scraping {app_id}:"):
            buat_chart(df, target_year)
            st.write(df)
        
            if download_format == "XLSX":
                # Clean the data to remove illegal characters
                cleaned_data = df.applymap(lambda x: "".join(char for char in str(x) if char.isprintable()))
        
                # Save the cleaned data to Excel
                cleaned_data.to_excel(f"hasil_scraping_{app_id}.xlsx", index=False)
        
                # Provide the download button for the cleaned Excel file
                st.download_button(label=f"Unduh XLSX ({len(reviews)} data)", data=open(f"hasil_scraping_{app_id}.xlsx", "rb").read(), key="xlsx_download", file_name=f"hasil_scraping_{app_id}.xlsx")
        
            elif download_format == "CSV":
                csv = df.to_csv(index=False)
        
                # Provide the download button for the CSV file
                st.download_button(label=f"Unduh CSV ({len(reviews)} data)", data=csv, key="csv_download", file_name=f"hasil_scraping_{app_id}.csv")
        
            elif download_format == "JSON":
                json_data = df.to_json(orient="records")
        
                # Provide the download button for the JSON file
                st.download_button(label=f"Unduh JSON ({len(reviews)} data)", data=json_data, key="json_download", file_name=f"hasil_scraping_{app_id}.json")
        
    else:
        st.info("Tidak ada data")
        
with tab3:
    @st.cache_resource
    def tab_3():
        st.header("Profile:")
        st.image('https://raw.githubusercontent.com/naufalnashif/naufalnashif.github.io/main/assets/img/my-profile-sidang-idCard-crop.JPG', caption='Naufal Nashif')
        st.subheader('Hello, nice to meet you !')
        # Tautan ke GitHub
        github_link = "https://github.com/naufalnashif/"
        st.markdown(f"GitHub: [{github_link}]({github_link})")
        
        # Tautan ke Instagram
        instagram_link = "https://www.instagram.com/naufal.nashif/"
        st.markdown(f"Instagram: [{instagram_link}]({instagram_link})")
        
        # Tautan ke Website
        website_link = "https://naufalnashif.netlify.app/"
        st.markdown(f"Website: [{website_link}]({website_link})")
    tab_3()
    
with tab4:
    @st.cache_resource   
    def tab_4():
        st.header("More:")
        more1, more2, more3 = st.columns(3)
        with more1 :
            st.image('https://raw.githubusercontent.com/naufalnashif/huggingface-repo/main/assets/img/sentiment-analysis-biskita.png', caption = 'Sentiment Analysis Web App')
            more1_link = "https://huggingface.co/spaces/naufalnashif/sentiment-analysis-ensemble-model"
            st.markdown(f"[{more1_link}]({more1_link})")
        with more2 :
            st.image('https://raw.githubusercontent.com/naufalnashif/huggingface-repo/main/assets/img/scraping-news-headline.png', caption = 'Scraping News Headline')
            more2_link = "https://huggingface.co/spaces/naufalnashif/scraping-news-headline"
            st.markdown(f"[{more2_link}]({more2_link})")
        with more3 :
            st.image('https://raw.githubusercontent.com/naufalnashif/huggingface-repo/main/assets/img/scraping-ecommerce.png', caption = 'Scraping Ecommerce Product')
            more3_link = "https://huggingface.co/spaces/naufalnashif/scraping-ecommerce-2023"
            st.markdown(f"[{more3_link}]({more3_link})")
    tab_4()
    
# Garis pemisah
st.divider()
st.write('Thank you for trying the demo!') 
st.caption(f'Made with ❤️ by :blue[Naufal Nashif] ©️ {yearNow}')