Update app.py

app.py

@@ -6,8 +6,11 @@ import streamlit as st
 import json
 import time
 
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
 @st.cache_data
-def scrape_e_commerce(nama_barang, num_items):
+def scrape_klikindomaret(nama_barang, num_items):
     products = []
     page = 1
     query = quote(nama_barang)
@@ -63,6 +66,80 @@ def scrape_e_commerce(nama_barang, num_items):
     my_bar.empty()
     return products
 
+@st.cache_data
+def scrape_shopee(nama_barang, num_items):
+    products = []
+    page = 1
+    query = quote(nama_barang)
+    progress_text = "Scraping in progress. Please wait."
+    my_bar = st.progress(0, text=progress_text)
+    url = f'https://shopee.co.id/search?keyword={nama_barang}&page={page}'
+    #path = ''
+    
+    #Customize chrome display
+    chrome_options = Options()
+    chrome_options.add_argument('--no-sandbox')
+    #chrome_options.add_argument('--headless')
+    chrome_options.add_argument('--disable-notifications')
+    chrome_options.add_argument('--disable-infobars')
+
+    while len(products) < num_items :
+        #Cek agar produk sesuai jumlah yang diminta
+        if len (products) > num_items :
+            products = products[:num_items]
+            break  
+        
+        #driver = webdriver.Chrome(executable_path = path, options = chrome_options)
+        driver = webdriver.Chrome(options = chrome_options)
+        driver.get(url)
+    
+        html = driver.execute_script("return document.getElementsByTagName('html')[0].innerHTML")
+        soup = BeautifulSoup(html, "html.parser")
+
+        product_list = soup.find_all('li', class ="col-xs-2-4 shopee-search-item-result__item" )
+        for product in product_list:
+            # Mencari tag <a> di dalam setiap tag <li>
+            a_tag = product.find_all('a', href=True)
+        
+            for product_info in a_tag:
+                # Mendapatkan URL dari atribut 'href'
+                product_href = product_info['href']
+                product_name = product.find('div', class_="ie3A+n bM+7UW Cve6sh").text.strip()
+                product_price = product.find('div', class="vioxXd rVLWG6").text.strip()
+                product_terjual = product.find('div', class="r6HknA uEPGHT").text.strip()
+                product_asal = product.find('div', class="zGGwiV").text.strip()
+
+                # Cek apakah ada harga sebelum diskon dan persentase diskon
+                #discount_element = product.find('span', class_='strikeout disc-price')
+                #discount_percentage = ""
+                #original_price = ""
+                #if discount_element:
+                #    discount_percentage = discount_element.find('span', class_='discount').text.strip()
+                #    original_price = discount_element.text.replace(discount_percentage, '').strip()
+                #else:
+                #    # Jika tidak ada diskon, set discount_percentage ke "0%" dan original_price ke product_price
+                #    discount_percentage = "0%"
+                #    original_price = product_price
+                #
+                product_link = f"https://shopee.co.id/{product_href}"
+                products.append({
+                    'product': product_name,
+                    #'original_price': original_price,
+                    #'discount_percentage': discount_percentage,
+                    'price': product_price,
+                    'terjual' : product_terjual,
+                    'asal' : product_asal,
+                    'link': product_link
+                })
+
+                prop = min(len(products)/num_items, 1)
+                my_bar.progress(prop, text=progress_text)
+                
+        
+        page += 1
+    time.sleep(1)   
+    my_bar.empty()
+    return products
 #---------------------------------------------------User Interface----------------------------------------------------------------------
 
 # Streamlit UI
@@ -70,7 +147,7 @@ st.title("Scraping E-Commerce")
 
 with st.expander("Settings :"):
     # Pilihan untuk memilih situs web
-    selected_site = st.selectbox("Pilih Situs Web :", ["klikindomaret.com", "shopee.co.id(under maintenance)"])
+    selected_site = st.selectbox("Pilih Situs Web :", ["klikindomaret.com", "shopee.co.id"])
     
     nama_barang = st.text_input("Masukkan Nama Barang :")
     num_items = st.number_input("Masukkan Estimasi Banyak Data :", min_value = 1, step = 1, placeholder="Type a number...")
@@ -88,21 +165,28 @@ if selected_site == "klikindomaret.com":
         if not nama_barang:
             st.error("Mohon isi Nama Barang.")
         else:
-            scraped_products = scrape_e_commerce(nama_barang, num_items)
+            scraped_products = scrape_klikindomaret(nama_barang, num_items)
             hidden_data = scraped_products  # Simpan data ke dalam variabel tersembunyi
             scraping_done = True  # Set scraping_done menjadi True
 
-if selected_site == "shopee.co.id(under maintenance)":
-    st.error("Sedang dalam pengembangan. Silahkan pilih situs yang lain")
+if selected_site == "shopee.co.id":
+    #st.error("Sedang dalam pengembangan. Silahkan pilih situs yang lain")
+    if st.button("Mulai Scraping"):
+        if not nama_barang:
+            st.error("Mohon isi Nama Barang.")
+        else:
+            scraped_products = scrape_shopee(nama_barang, num_items)
+            hidden_data = scraped_products  # Simpan data ke dalam variabel tersembunyi
+            scraping_done = True  # Set scraping_done menjadi True
 
 
 
 
 
 # Simpan DataFrame ke dalam file
-output_file = f"scraped_{nama_barang}.xlsx"
-output_file_csv = f"scraped_{nama_barang}.csv"
-output_file_json = f"scraped_{nama_barang}.json"
+output_file = f"scraped_{selected_site}_{nama_barang}.xlsx"
+output_file_csv = f"scraped_{selected_site}_{nama_barang}.csv"
+output_file_json = f"scraped_{selected_site}_{nama_barang}.json"
 
 
 #---------------------------------------------------Download File & Hasil Scraping----------------------------------------------------------------------