Spaces:
Sleeping
Sleeping
#---------------------------------------------------Requirements---------------------------------------------------------------------- | |
import streamlit as st | |
import pandas as pd | |
import numpy as np | |
import re | |
import json | |
import matplotlib.pyplot as plt | |
import seaborn as sns | |
from wordcloud import WordCloud | |
import requests | |
from bs4 import BeautifulSoup | |
from datetime import date | |
import time | |
from collections import Counter | |
import nltk | |
from nltk.corpus import stopwords | |
#---------------------------------------------------Scraping Function---------------------------------------------------------------------- | |
def scrape_cnbc_data(query, date, jumlah): | |
data = [] | |
page = 1 | |
progress_text = "Scraping in progress. Please wait." | |
my_bar = st.progress(len(data), text=progress_text) | |
for percent_complete in range(jumlah): | |
if len(data) > jumlah: | |
data = data[:jumlah] | |
break | |
prop = min(len(data) / jumlah, 1) | |
my_bar.progress(prop, text=progress_text) | |
base_url = f"https://www.cnbcindonesia.com/search?query={query}&p={page}&kanal=&tipe=artikel&date={date}" | |
headers = { | |
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" | |
} | |
response = requests.get(base_url, headers=headers) | |
soup = BeautifulSoup(response.content, 'html.parser') | |
articles = soup.find_all('article') | |
if not articles: | |
break | |
for article in articles: | |
title = article.find('h2').text.strip() | |
link = article.find('a')['href'] | |
date = article.find('span', class_='date').text.strip() | |
data.append({ | |
'date': date, | |
'judul-berita': title, | |
'link-berita': link, | |
}) | |
page += 1 | |
time.sleep(1) | |
my_bar.empty() | |
return data | |
def scrape_detik_news(query, jumlah): | |
site_id = 2 | |
data = [] | |
page = 1 | |
progress_text = "Scraping in progress. Please wait." | |
my_bar = st.progress(len(data), text=progress_text) | |
for percent_complete in range(jumlah): | |
if len(data) > jumlah: | |
data = data[:jumlah] | |
break | |
prop = min(len(data) / jumlah, 1) | |
my_bar.progress(prop, text=progress_text) | |
base_url = "https://www.detik.com/search/searchall" | |
params = { | |
"query": query, | |
"siteid": site_id, | |
"page": page | |
} | |
response = requests.get(base_url, params=params) | |
soup = BeautifulSoup(response.content, "html.parser") | |
articles = soup.find_all("article") | |
if not articles: | |
break | |
for article in articles: | |
date = article.find("span", class_="date").text.strip() | |
title = article.find("h2", class_="title").text.strip() | |
link = article.find("a")["href"] | |
data.append({"date": date, "judul-berita": title, "link-berita": link}) | |
page += 1 | |
time.sleep(1) | |
my_bar.empty() | |
return data | |
#---------------------------------------------------Data Cleaning (RegEx)---------------------------------------------------------------------- | |
def clean_text(text): | |
# Tahap-1: Menghapus karakter non-ASCII | |
text = re.sub(r'[^\x00-\x7F]+', '', text) | |
# Tahap-2: Menghapus URL | |
text = re.sub(r'http[s]?://.[a-zA-Z0-9./_?=%&#+!]+', '', text) | |
text = re.sub(r'pic.twitter.com?.[a-zA-Z0-9./_?=%&#+!]+', '', text) | |
# Tahap-3: Menghapus mentions | |
text = re.sub(r'@[\w]+', '', text) | |
# Tahap-4: Menghapus hashtag | |
text = re.sub(r'#([\w]+)', '', text) | |
# Tahap-5 Menghapus 'amp' yang menempel pada '&' dan 'gt' yang menempel pada '&' | |
text = re.sub(r'&|>', '', text) | |
# Tahap-6: Menghapus karakter khusus (simbol) | |
text = re.sub(r'[!$%^&*@#()_+|~=`{}\[\]%\-:";\'<>?,./]', '', text) | |
# Tahap-7: Menghapus angka | |
text = re.sub(r'[0-9]+', '', text) | |
# Tahap-8: Menggabungkan spasi ganda menjadi satu spasi | |
text = re.sub(' +', ' ', text) | |
# Tahap-9: Menghapus spasi di awal dan akhir kalimat | |
text = text.strip() | |
# Tahap-10: Konversi teks ke huruf kecil | |
text = text.lower() | |
# Tahap-11: koreksi duplikasi tiga karakter beruntun atau lebih (contoh. yukkk) | |
# text = re.sub(r'([a-zA-Z])\1\1', '\\1', text) | |
#text = re.sub(r'(.)(\1{2,})', r'\1\1', text) | |
text = re.sub(r'(\w)\1{2,}', r'\1', text) | |
return text | |
#---------------------------------------------------Normalisasi---------------------------------------------------------------------- | |
# Membaca kamus kata gaul Salsabila | |
kamus_path = '_json_colloquial-indonesian-lexicon.txt' # Ganti dengan path yang benar | |
with open(kamus_path) as f: | |
data = f.read() | |
lookp_dict = json.loads(data) | |
# Dict kata gaul saya sendiri yang tidak masuk di dict Salsabila | |
kamus_sendiri_path = 'kamus_gaul_custom.txt' | |
with open(kamus_sendiri_path) as f: | |
kamus_sendiri = f.read() | |
kamus_gaul_baru = json.loads(kamus_sendiri) | |
# Menambahkan dict kata gaul baru ke kamus yang sudah ada | |
lookp_dict.update(kamus_gaul_baru) | |
# Fungsi untuk normalisasi kata gaul | |
def normalize_slang(text, slang_dict): | |
words = text.split() | |
normalized_words = [slang_dict.get(word, word) for word in words] | |
return ' '.join(normalized_words) | |
#---------------------------------------------------NLTK Remove Stopwords---------------------------------------------------------------------- | |
# Inisialisasi stopwords bahasa Indonesia | |
nltk.download("stopwords") | |
stop_words = set(stopwords.words("indonesian")) | |
def remove_stopwords(text, stop_words): | |
# Pecah teks menjadi kata-kata | |
words = text.split() | |
# Hapus stopwords bahasa Indonesia | |
words = [word for word in words if word not in stop_words] | |
return " ".join(words) | |
#---------------------------------------------------User Interface---------------------------------------------------------------------- | |
# Streamlit UI | |
st.title("Aplikasi Web Scraping CNBC / Detik.com & Explorasi Data") | |
# Pilihan untuk memilih situs web | |
selected_site = st.selectbox("Pilih Situs Web :", ["CNBC Indonesia", "Detik.com"]) | |
query = st.text_input("Masukkan Query :") | |
jumlah = st.number_input("Masukkan Estimasi Banyak Data :", min_value = 1, step = 1, placeholder="Type a number...") | |
date = date.today() | |
download_format = st.selectbox("Pilih Format Unduhan :", ["XLSX", "CSV", "JSON", "TXT"]) | |
st.info('Tekan "Mulai Scraping" kembali jika tampilan menghilang ', icon="ℹ️") | |
# Variabel tersembunyi untuk menyimpan hasil scraping | |
hidden_data = [] | |
scraping_done = False # Tambahkan variabel ini | |
#---------------------------------------------------CNBC Indonesia---------------------------------------------------------------------- | |
if selected_site == "CNBC Indonesia": | |
if st.button("Mulai Scraping"): | |
if not query: | |
st.error("Mohon isi query.") | |
else: | |
data_df = scrape_cnbc_data(query, date.strftime("%Y/%m/%d"), jumlah) | |
hidden_data = data_df # Simpan data ke dalam variabel tersembunyi | |
scraping_done = True # Set scraping_done menjadi True | |
#---------------------------------------------------Eksplorasi Data-------------------------------------------------- | |
df = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]) | |
texts = df["judul-berita"] | |
# Initialize results | |
results = [] | |
# Process the text data | |
for text in texts: | |
cleaned_text = clean_text(text) | |
norm_slang_text = normalize_slang(cleaned_text, lookp_dict) | |
tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words) | |
results.append((text, cleaned_text, norm_slang_text, tanpa_stopwords)) | |
# Membagi tampilan menjadi dua kolom | |
columns = st.columns(2) | |
# Inisialisasi all_texts di luar blok with columns[0] | |
all_texts = "" | |
# Kolom pertama untuk Word Cloud | |
with columns[0]: | |
if results: | |
all_texts = [result[3] for result in results if result[3] is not None and not pd.isna(result[3])] | |
all_texts = " ".join(all_texts) | |
st.subheader("Word Cloud") | |
if all_texts: | |
wordcloud = WordCloud(width=800, height=660, background_color='white', | |
colormap='Purples', | |
contour_color='black', | |
contour_width=2, | |
mask=None).generate(all_texts) | |
st.image(wordcloud.to_array()) | |
else: | |
st.write("Tidak ada data untuk ditampilkan.") | |
# Kolom kedua untuk Most Comon Words | |
with columns[1]: | |
st.subheader("Most Common Words") | |
if all_texts: | |
word_counts = Counter(all_texts.split()) | |
most_common_words = word_counts.most_common(5) | |
words, counts = zip(*most_common_words) | |
fig, ax = plt.subplots(figsize=(10, 6)) | |
ax.bar(words, counts) | |
ax.set_xlabel("Kata-kata") | |
ax.set_ylabel("Jumlah") | |
ax.set_title("Kata-kata Paling Umum") | |
ax.tick_params(axis='x', rotation=45) | |
st.pyplot(fig) | |
else: | |
st.write("Tidak ada data untuk ditampilkan dalam Word Cloud.") | |
if not hidden_data: | |
st.warning(f"Tidak ada data pada query '{query}'", icon="⚠️") | |
#---------------------------------------------------Detik.com---------------------------------------------------------------------- | |
elif selected_site == "Detik.com": | |
if st.button("Mulai Scraping"): | |
if not query: | |
st.error("Mohon isi query.") | |
else: | |
data_df = scrape_detik_news(query = query, jumlah = jumlah) | |
hidden_data = data_df # Simpan data ke dalam variabel tersembunyi | |
scraping_done = True | |
#---------------------------------------------------Eksplorasi Data-------------------------------------------------- | |
df = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]) | |
texts = df["judul-berita"] | |
# Initialize results | |
results = [] | |
# Process the text data | |
for text in texts: | |
cleaned_text = clean_text(text) | |
norm_slang_text = normalize_slang(cleaned_text, lookp_dict) | |
tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words) | |
results.append((text, cleaned_text, norm_slang_text, tanpa_stopwords)) | |
# Membagi tampilan menjadi dua kolom | |
columns = st.columns(2) | |
# Inisialisasi all_texts di luar blok with columns[0] | |
all_texts = "" | |
# Kolom pertama untuk Word Cloud | |
with columns[0]: | |
if results: | |
all_texts = [result[3] for result in results if result[3] is not None and not pd.isna(result[3])] | |
all_texts = " ".join(all_texts) | |
st.subheader("Word Cloud") | |
if all_texts: | |
wordcloud = WordCloud(width=800, height=660, background_color='white', | |
colormap='Purples', | |
contour_color='black', | |
contour_width=2, | |
mask=None).generate(all_texts) | |
st.image(wordcloud.to_array()) | |
else: | |
st.write("Tidak ada data untuk ditampilkan.") | |
# Kolom kedua untuk Most Common Words | |
with columns[1]: | |
st.subheader("Most Common Words") | |
if all_texts: | |
word_counts = Counter(all_texts.split()) | |
most_common_words = word_counts.most_common(5) | |
words, counts = zip(*most_common_words) | |
fig, ax = plt.subplots(figsize=(10, 6)) | |
ax.bar(words, counts) | |
ax.set_xlabel("Kata-kata") | |
ax.set_ylabel("Jumlah") | |
ax.set_title("Kata-kata Paling Umum") | |
ax.tick_params(axis='x', rotation=45) | |
st.pyplot(fig) | |
else: | |
st.write("Tidak ada data untuk ditampilkan dalam Word Cloud.") | |
if not hidden_data: | |
st.warning(f"Tidak ada data pada query '{query}'", icon="⚠️") | |
#---------------------------------------------------Download File & Hasil Scraping---------------------------------------------------------------------- | |
# Tampilkan hasil scraping | |
if scraping_done: | |
if hidden_data: | |
if download_format == "XLSX": | |
st.subheader("Hasil Scraping") | |
st.write(pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"])) | |
df = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]) | |
df.to_excel("hasil_scraping.xlsx", index=False) | |
st.download_button(label=f"Unduh XLSX ({len(hidden_data)} data)", data=open("hasil_scraping.xlsx", "rb").read(), key="xlsx_download", file_name="hasil_scraping.xlsx") | |
elif download_format == "CSV": | |
st.subheader("Hasil Scraping") | |
st.write(pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"])) | |
df = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]) | |
csv = df.to_csv(index=False) | |
st.download_button(label=f"Unduh CSV ({len(hidden_data)} data)", data=csv, key="csv_download", file_name="hasil_scraping.csv") | |
elif download_format == "JSON": | |
st.subheader("Hasil Scraping") | |
st.write(pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"])) | |
json_data = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]).to_json(orient="records") | |
st.download_button(label=f"Unduh JSON ({len(hidden_data)} data)", data=json_data, key="json_download", file_name="hasil_scraping.json") | |
elif download_format == "TXT": | |
st.subheader("Hasil Scraping") | |
st.write(pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"])) | |
text_data = "\n".join([f"{row['date']} - {row['judul-berita']} - {row['link-berita']}" for row in hidden_data]) | |
st.download_button(label=f"Unduh TXT ({len(hidden_data)} data)", data=text_data, key="txt_download", file_name="hasil_scraping.txt") | |
if not scraping_done: | |
st.write("Tidak ada data untuk diunduh.") | |
st.divider() | |
github_link = "https://github.com/naufalnashif/" | |
st.markdown(f"GitHub: [{github_link}]({github_link})") | |
instagram_link = "https://www.instagram.com/naufal.nashif/" | |
st.markdown(f"Instagram: [{instagram_link}]({instagram_link})") | |
st.write('Terima kasih telah mencoba demo ini!') | |