Upload 3 files

main.py

@@ -0,0 +1,78 @@
+import os
+import requests
+from bs4 import BeautifulSoup
+import sqlite3
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+
+# Database setup
+def create_database():
+    conn = sqlite3.connect('courses.db')
+    c = conn.cursor()
+    # Create table with an additional column 'price' to indicate free or paid
+    c.execute('''CREATE TABLE IF NOT EXISTS courses
+                 (id INTEGER PRIMARY KEY, title TEXT, description TEXT, price TEXT)''')
+    conn.commit()
+    conn.close()
+
+
+# Web scraping function to get course data from a specific page
+def scrape_courses_from_page(page_number):
+    url = f"https://courses.analyticsvidhya.com/collections/courses?page={page_number}"
+    response = requests.get(url)
+    soup = BeautifulSoup(response.text, 'html.parser')
+
+    courses = []
+
+    # Find all course items from the products__list
+    course_items = soup.find_all('li', class_='products__list-item')
+
+    for course_item in course_items:
+        # Extract the course title
+        title_tag = course_item.find('h3')
+        title = title_tag.text.strip() if title_tag else 'No title available'
+
+        # Extract course price
+        price_tag = course_item.find('span', class_='course-card__price')
+        price = price_tag.text.strip() if price_tag else "Price not available"
+
+        # Description is not always explicitly provided, using course body for more details
+        description_tag = course_item.find('h4')
+        description = description_tag.text.strip() if description_tag else 'No description available'
+
+        # Append course details (title, description, price)
+        courses.append((title, description, price))
+
+    return courses
+
+
+# Scrape all pages (total 8 pages) and insert data into the database
+def scrape_all_pages():
+    all_courses = []
+
+    # Loop through pages 1 to 8
+    for page in range(1, 9):
+        print(f"Scraping page {page}...")
+        courses = scrape_courses_from_page(page)
+        all_courses.extend(courses)
+
+    return all_courses
+
+
+# Insert scraped data into the database
+def insert_data_to_db(courses):
+    conn = sqlite3.connect('courses.db')
+    c = conn.cursor()
+    c.executemany('INSERT INTO courses (title, description, price) VALUES (?, ?, ?)', courses)
+    conn.commit()
+    conn.close()
+
+
+if __name__ == "__main__":
+    create_database()
+    all_courses = scrape_all_pages()
+    insert_data_to_db(all_courses)
+    print(f"Data from all pages has been successfully scraped and inserted into the database.")

requirements.txt

@@ -0,0 +1,7 @@
+requests
+beautifulsoup4
+gradio
+streamlit
+sqlite3
+python-dotenv
+openai