#---------------------------------------------------Requirements---------------------------------------------------------------------- import streamlit as st import pandas as pd import random import numpy as np import re import json import matplotlib.pyplot as plt import seaborn as sns from wordcloud import WordCloud import requests from bs4 import BeautifulSoup from datetime import date import time from collections import Counter import nltk from nltk.corpus import stopwords #---------------------------------------------------Scraping Function---------------------------------------------------------------------- @st.cache_data def scrape_cnbc_data(query, date, jumlah, param_kosong): data = [] page = 1 progress_text = "Scraping in progress. Please wait." my_bar = st.progress(len(data), text=progress_text) while len (data) < jumlah : try : url = f"https://www.cnbcindonesia.com/search?query={query}&p={page}&kanal=&tipe=artikel&date={date}" user_agents = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", ] # Mendapatkan User-Agent acak random_user_agent = random.choice(user_agents) # Menggunakan User-Agent dalam permintaan HTTP headers = { "User-Agent": random_user_agent, "Accept-Language": "en-US,en;q=0.5" } timeout = 10 response = requests.get(url, headers=headers, timeout = timeout) soup = BeautifulSoup(response.content, 'html.parser') articles = soup.find_all('article') if not articles: break for article in articles: title = article.find('h2').text.strip() link = article.find('a')['href'] category = article.find('span', class_ = 'label').text.strip() date_category = article.find('span', class_='date').text.strip() text_parts = date_category.split(' - ') date = text_parts[1].strip() data.append({ 'category': category, 'date': date, 'judul-berita': title, 'link-berita': link, }) if len(data) > jumlah: data = data[:jumlah] break prop = min(len(data) / jumlah, 1) my_bar.progress(prop, text=progress_text) page += 1 except requests.exceptions.RequestException as e: st.error(f"An error occurred: {e}") break time.sleep(1) my_bar.empty() return data @st.cache_data def scrape_detik_news(query, date, jumlah, param_kosong): start_page = 1 base_url = "https://www.detik.com/search/searchall" data = [] progress_text = "Scraping in progress... Please wait..." my_bar = st.progress(len(data), text=progress_text) timeout = 10 while len(data) < jumlah: try: params = { "query": query, "siteid": 2, "sortby": "time", "page": start_page } url = f'https://www.detik.com/search/searchall?query={query}&siteid=2&sortby=time&page={start_page}' # Daftar beberapa User-Agent user_agents = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", ] # Mendapatkan User-Agent acak random_user_agent = random.choice(user_agents) # Menggunakan User-Agent dalam permintaan HTTP headers = { "User-Agent": random_user_agent, "Accept-Language": "en-US,en;q=0.5" } response = requests.get(url, headers=headers, timeout = timeout) response.raise_for_status() soup = BeautifulSoup(response.text, 'html.parser') articles = soup.find_all('article') if not articles : break for article in articles : title = article.find('h2').text.strip() link = article.find('a')['href'] category = article.find('span', class_='category').text date_category = article.find('span', class_='date').text date = date_category.replace(category, '').strip() data.append({ 'category': category, 'date': date, 'judul-berita': title, 'link-berita': link, }) if len(data) >= jumlah: data = data[:jumlah] break prop = min(len(data) / jumlah, 1) my_bar.progress(prop, text=progress_text) start_page += 1 except requests.exceptions.RequestException as e: st.error(f"An error occurred: {e}") break time.sleep(1) my_bar.empty() return data @st.cache_data def scrape_viva_data(query, date, jumlah, param_kosong): data = [] page = 1 progress_text = "Scraping in progress. Please wait." my_bar = st.progress(len(data), text=progress_text) while len (data) < jumlah : try : url = f"https://www.viva.co.id/search?q={query}" user_agents = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", ] # Mendapatkan User-Agent acak random_user_agent = random.choice(user_agents) # Menggunakan User-Agent dalam permintaan HTTP headers = { "User-Agent": random_user_agent, "Accept-Language": "en-US,en;q=0.5" } timeout = 10 response = requests.get(url, headers=headers, timeout = timeout) soup = BeautifulSoup(response.content, 'html.parser') articles = soup.find_all('div', class_='card-box ft240 margin-bottom-sm') if not articles : break for article in articles : title = article.find('h2', class_='title').text link = article.find('a')['href'] category_element = article.find('span', class_="kanal cl-dark") category = category_element.text.strip() if category_element else None date_element = article.find('h4', class_="date") date_before = date_element.text.strip() if date_element else None date = date_before.replace(category, '') data.append({ 'category': category, 'date': date, 'judul-berita': title, 'link-berita': link, }) if len(data) > jumlah: data = data[:jumlah] break prop = min(len(data) / jumlah, 1) my_bar.progress(prop, text=progress_text) page += 1 except requests.exceptions.RequestException as e: st.error(f"An error occurred: {e}") break time.sleep(1) my_bar.empty() return data @st.cache_data def scrape_tempo_data(query, date, jumlah, selected_channel): data = [] domain = 1 max_domains = 5 progress_text = "Scraping in progress. Please wait." my_bar = st.progress(len(data), text=progress_text) # List of channel values default_channels = { 'All(Latest Only)': '', 'Nasional': '20', 'Metro': '19', 'Dunia': '5', 'Bisnis': '1', 'Bola': '21', 'Sport': '33', 'Gaya': '9', 'Seleb': '32', 'Cantik': '2', 'Tekno': '34', 'Otomotif': '23', 'Travel': '35', 'Blog': '43', 'Difabel': '44', 'Ramadan': '30', 'Kolom': '14', 'Fokus': '8', 'Creative Lab': '47', 'Event': '62', 'Data': '65', 'Cek Fakta': '66', 'Newsletter': '63', 'Inforial': '12' } # Ubah channels sesuai dengan selected_channel if selected_channel != 'Defaults' and selected_channel in default_channels: channels = {selected_channel: default_channels[selected_channel]} else: channels = default_channels seen_titles = set() # Set untuk melacak judul berita yang sudah muncul try: while len(data) < jumlah and domain <= max_domains: for kanal, value in channels.items(): url = f"https://www.tempo.co/search?waktu={waktu}&kanal={value}&subkanal=&domain={domain}&q={query}" user_agents = [ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.1.2 Safari/605.1.15", ] # Get a random User-Agent random_user_agent = random.choice(user_agents) # Use User-Agent in the HTTP request headers = { "User-Agent": random_user_agent, "Accept-Language": "en-US,en;q=0.5" } timeout = 10 response = requests.get(url, headers=headers, timeout=timeout) soup = BeautifulSoup(response.text, 'html.parser') articles = soup.find_all('div', class_='card-box ft240 margin-bottom-sm') if not articles: break for article in articles: title = article.find('h2', class_='title').text # Hanya proses artikel yang belum pernah ditemui if title not in seen_titles: link = article.find('a')['href'] category_element = article.find('span', class_="kanal cl-dark") category = category_element.text.strip() if category_element else None date_element = article.find('h4', class_="date") date_before = date_element.text.strip() if date_element else None date = date_before.replace(category, '') data.append({ 'category': category, 'kanal' : kanal, 'date': date, 'judul-berita': title, 'link-berita': link, }) seen_titles.add(title) # Tambahkan judul berita ke set if len(data) >= jumlah: break if len(data) >= jumlah: break prop = min(len(data) / jumlah, 1) my_bar.progress(prop, text=progress_text) domain += 1 except requests.exceptions.RequestException as e: st.error(f"An error occurred: {e}") time.sleep(1) my_bar.empty() return data #---------------------------------------------------Data Cleaning (RegEx)---------------------------------------------------------------------- def clean_text(text): # Pastikan text adalah string if not isinstance(text, str): text = str(text) # Tahap-1: Menghapus karakter non-ASCII text = re.sub(r'[^\x00-\x7F]+', '', text) # Tahap-2: Menghapus URL text = re.sub(r'http[s]?://.[a-zA-Z0-9./_?=%&#+!]+', '', text) text = re.sub(r'pic.twitter.com?.[a-zA-Z0-9./_?=%&#+!]+', '', text) # Tahap-3: Menghapus mentions text = re.sub(r'@[\w]+', '', text) # Tahap-4: Menghapus hashtag text = re.sub(r'#([\w]+)', '', text) # Tahap-5 Menghapus 'amp' yang menempel pada '&' dan 'gt' yang menempel pada '&' text = re.sub(r'&|>', '', text) # Tahap-6: Menghapus karakter khusus (simbol) text = re.sub(r'[!$%^&*@#()_+|~=`{}\[\]%\-:";\'<>?,./]', '', text) # Tahap-7: Menghapus angka text = re.sub(r'[0-9]+', '', text) # Tahap-8: Menggabungkan spasi ganda menjadi satu spasi text = re.sub(' +', ' ', text) # Tahap-9: Menghapus spasi di awal dan akhir kalimat text = text.strip() # Tahap-10: Konversi teks ke huruf kecil text = text.lower() # Tahap-11: koreksi duplikasi tiga karakter beruntun atau lebih (contoh. yukkk) # text = re.sub(r'([a-zA-Z])\1\1', '\\1', text) #text = re.sub(r'(.)(\1{2,})', r'\1\1', text) text = re.sub(r'(\w)\1{2,}', r'\1', text) return text #---------------------------------------------------Normalisasi---------------------------------------------------------------------- # Membaca kamus kata gaul Salsabila kamus_path = '_json_colloquial-indonesian-lexicon.txt' # Ganti dengan path yang benar with open(kamus_path) as f: data = f.read() lookp_dict = json.loads(data) # Dict kata gaul saya sendiri yang tidak masuk di dict Salsabila kamus_sendiri_path = 'kamus_gaul_custom.txt' with open(kamus_sendiri_path) as f: kamus_sendiri = f.read() kamus_gaul_baru = json.loads(kamus_sendiri) # Menambahkan dict kata gaul baru ke kamus yang sudah ada lookp_dict.update(kamus_gaul_baru) # Fungsi untuk normalisasi kata gaul def normalize_slang(text, slang_dict): words = text.split() normalized_words = [slang_dict.get(word, word) for word in words] return ' '.join(normalized_words) #---------------------------------------------------NLTK Remove Stopwords---------------------------------------------------------------------- # Inisialisasi stopwords bahasa Indonesia nltk.download("stopwords") stop_words = set(stopwords.words("indonesian")) def remove_stopwords(text, stop_words): # Pecah teks menjadi kata-kata words = text.split() # Hapus stopwords bahasa Indonesia words = [word for word in words if word not in stop_words] return " ".join(words) def preprocessing_data(hidden_data): # Initialize results results_prep = [] df = pd.DataFrame(hidden_data) texts = df["judul-berita"] # Process the text data for text in texts: cleaned_text = clean_text(text) norm_slang_text = normalize_slang(cleaned_text, lookp_dict) tanpa_stopwords = remove_stopwords(norm_slang_text, stop_words) results_prep.append({ 'judul-berita': text, 'cleaned-text' : cleaned_text, 'normalisasi-text' : norm_slang_text, 'stopwords-remove' : tanpa_stopwords, }) return results_prep def eksplorasi_data(selected_options, results, colormap, words): # Kolom pertama untuk Word Cloud if 'Hasil EDA' in selected_options: # Membagi tampilan menjadi dua kolom columns = st.columns(2) all_texts = "" with columns[0]: if results: all_texts = all_texts = [result.get('stopwords-remove') for result in results if pd.notna(result.get('stopwords-remove'))] all_texts = " ".join(all_texts) st.subheader("Word Cloud") if all_texts: wordcloud = WordCloud(width=800, height=500, background_color='white', colormap=colormap, contour_color='black', contour_width=2, mask=None).generate(all_texts) st.image(wordcloud.to_array()) # Kolom kedua untuk Most Common Words with columns[1]: st.subheader("Most Common Words") if all_texts: word_counts = Counter(all_texts.split()) most_common_words = word_counts.most_common(words) words, counts = zip(*most_common_words) fig, ax = plt.subplots(figsize=(10, 6)) ax.bar(words, counts) ax.set_xlabel("Kata-kata") ax.set_ylabel("Jumlah") ax.set_title("Kata-kata Paling Umum") ax.tick_params(axis='x', rotation=45) st.pyplot(fig) @st.cache_data def scrape_and_explore_data(_scrape_function, query, date, jumlah, selected_options, colormap, words, param): data_df = _scrape_function(query, date, jumlah, param) hidden_data = data_df scraping_done = True results = preprocessing_data(hidden_data) # Eksplorasi Data eksplorasi_data(selected_options, results, colormap, words) return hidden_data, scraping_done, results #---------------------------------------------------User Interface---------------------------------------------------------------------- # Streamlit UI st.title("Aplikasi Web Scraping & Explorasi Data") with st.expander("Scraping Settings :"): # Pilihan untuk memilih situs web selected_site = st.selectbox("Pilih Situs Web :", ["CNBC Indonesia", "Detik.com", "Viva.co.id", "Tempo.co", "Liputan6.com"]) if selected_site == "Tempo.co": waktu = st.selectbox("Pilih Rentang Waktu :", ["1tahun", "1bulan", "1minggu", "1hari", "6jam"]) selected_channel = st.selectbox("Pilih Kanal :", ['Defaults','All(Latest Only)', 'Nasional', 'Metro', 'Dunia', 'Bisnis', 'Bola', 'Sport', 'Gaya', 'Seleb', 'Cantik', 'Tekno', 'Otomotif', 'Travel', 'Blog', 'Difabel', 'Ramadan', 'Kolom', 'Fokus', 'Creative Lab', 'Event', 'Data', 'Cek Fakta', 'Newsletter', 'Inforial']) query = st.text_input("Masukkan Query :").replace(' ', '+') jumlah = st.number_input("Masukkan Estimasi Banyak Data :", min_value = 1, step = 1, placeholder="Type a number...") date = date.today() download_format = st.selectbox("Pilih Format Unduhan :", ["XLSX", "CSV", "JSON", "TXT"]) param_kosong = [] with st.expander("Preference Settings :"): selected_options = st.multiselect( 'Pilih tampilan:', ['Hasil Scraping', 'Hasil Preprocessing', 'Hasil EDA'], ["Hasil Scraping", "Hasil EDA"] ) if "Hasil EDA" in selected_options: colormap = st.selectbox("Pilih Warna Wordclouds :", ["Greys", "Purples", "Blues", "Greens", "Oranges", "Reds", "YlOrBr", "YlOrRd", "OrRd", "PuRd", "RdPu", "BuPu", "GnBu", "PuBu", "YlGnBu", "PuBuGn", "BuGn", "YlGn"]) words = st.number_input("Masukkan Jumlah Most Common Words :", min_value = 1, max_value = 15, step = 1, value = 10, placeholder="Type a number...") else : colormap = "Greys" words = 10 st.info('Tekan "Mulai Scraping" kembali jika tampilan menghilang ', icon="ℹ️") #------------------------------------------------------------Bakcend---------------------------------------------------------------------------------- # Variabel tersembunyi untuk menyimpan hasil scraping hidden_data = [] scraping_done = False # Tambahkan variabel ini if st.button("Mulai Scraping"): if not query: st.error("Mohon isi query.") else: # CNBC Indonesia if selected_site == "CNBC Indonesia": hidden_data, scraping_done, results = scrape_and_explore_data(scrape_cnbc_data, query, date.strftime("%Y/%m/%d"), jumlah, selected_options, colormap, words, param_kosong) # Detik.com elif selected_site == "Detik.com": hidden_data, scraping_done, results = scrape_and_explore_data(scrape_detik_news, query, date, jumlah, selected_options, colormap, words, param_kosong) # Viva.co.id elif selected_site == "Viva.co.id": st.warning("Masih dalam penegmbangan, silahkan gunakan situs yang lain.") hidden_data, scraping_done, results = scrape_and_explore_data(scrape_viva_data, query, date, jumlah, selected_options, colormap, words, param_kosong) # Tempo.co elif selected_site == "Tempo.co": st.warning("Masih dalam penegmbangan, silahkan gunakan situs yang lain.") hidden_data, scraping_done, results = scrape_and_explore_data(scrape_tempo_data, query, waktu, jumlah, selected_options, colormap, words, selected_channel) # Liputan6.com elif selected_site == "Liputan6.com": st.error("Belum bisa dipakai.") #---------------------------------------------------Download File & Hasil Scraping---------------------------------------------------------------------- # Tampilkan hasil scraping if scraping_done: if hidden_data: df = pd.DataFrame(hidden_data) df_prep = pd.DataFrame(results) # Menampilkan hasil sentimen dalam kotak yang dapat diperluas if 'Hasil Scraping' in selected_options: with st.expander(f"Hasil Scraping {selected_site} :"): st.write(df) if 'Hasil Preprocessing' in selected_options: with st.expander(f"Hasil Preprocessing Data :"): st.write(df_prep) if download_format == "XLSX": df.to_excel(f"hasil_scraping_{query}.xlsx", index=False) df_prep.to_excel(f"hasil_preprocess_{query}.xlsx", index=False) st.download_button(label=f"Unduh Hasil Scraping XLSX ({len(hidden_data)} data)", data=open(f"hasil_scraping_{query}.xlsx", "rb").read(), key="xlsx_download", file_name=f"hasil_scraping_{query}.xlsx") st.download_button(label=f"Unduh Hasil Preprocess XLSX ({len(results)} data)", data=open(f"hasil_preprocess_{query}.xlsx", "rb").read(), key="xlsx_download_2", file_name=f"hasil_preprocess_{query}.xlsx") elif download_format == "CSV": csv = df.to_csv(index=False) csv_prep = df_prep.to_csv(index = False) st.download_button(label=f"Unduh Hasil Scraping CSV ({len(hidden_data)} data)", data=csv, key="csv_download", file_name=f"hasil_scraping_{query}.csv") st.download_button(label=f"Unduh Hasil Preprocess CSV ({len(results)} data)", data=csv_prep, key="csv_download_2", file_name=f"hasil_preprocess_{query}.csv") elif download_format == "JSON": json_data = pd.DataFrame(hidden_data, columns=["date", "judul-berita", "link-berita"]).to_json(orient="records") json_data_prep = pd.DataFrame(results, columns=["Teks", "Cleaned Text", "Norm Text", "Tanpa Stopwords"]).to_json(orient="records") st.download_button(label=f"Unduh Hasil Scraping JSON ({len(hidden_data)} data)", data=json_data, key="json_download", file_name=f"hasil_scraping_{query}.json") st.download_button(label=f"Unduh Hasil Preprocess JSON ({len(results)} data)", data=json_data_prep, key="json_download_2", file_name=f"hasil_preprocess_{query}.json") elif download_format == "TXT": text_data = "\n".join([f"{row['date']} - {row['judul-berita']} - {row['link-berita']}" for row in hidden_data]) st.download_button(label=f"Unduh Hasil Scraping TXT ({len(hidden_data)} data)", data=text_data, key="txt_download", file_name=f"hasil_scraping_{query}.txt") if not hidden_data: st.warning(f"Tidak ada data pada query '{query}'", icon="⚠️") if not scraping_done: st.write("Tidak ada data untuk diunduh.") st.divider() github_link = "https://github.com/naufalnashif/" st.markdown(f"GitHub: [{github_link}]({github_link})") instagram_link = "https://www.instagram.com/naufal.nashif/" st.markdown(f"Instagram: [{instagram_link}]({instagram_link})") st.write('Terima kasih telah mencoba demo ini!')