Do not change the logic or anything in this, only make this change : make if go through 4 categories instead of only one, national : national,international,business,sportsand make it in series go to another category only when all the required pages in one are completed : import requests
from bs4 import BeautifulSoup
import pandas as pd
import datetime
import time
import logging
import sys

# ------------------------ Configuration ------------------------

# Set up logging
logging.basicConfig(
    filename='scrape_air_news.log',
    level=logging.INFO,
    format='%(asctime)s:%(levelname)s:%(message)s'
)

# Maximum number of consecutive skips before stopping the scraper
# Not needed now since we stop immediately on first mismatch
# MAX_CONSECUTIVE_SKIPS = 3

# ------------------------ Functions ------------------------

def get_soup(url):
    """
    Fetches the HTML content of a given URL and returns a BeautifulSoup object.
    """
    try:
        headers = {
            'User-Agent': (
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                'AppleWebKit/537.36 (KHTML, like Gecko) '
                'Chrome/58.0.3029.110 Safari/537.3'
            )
        }
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        return BeautifulSoup(response.text, 'html.parser')
    except requests.exceptions.RequestException as e:
        logging.error(f"Error fetching URL {url}: {e}")
        print(f"Error fetching URL {url}: {e}")
        return None

def scrape_article_content(article_url):
    """
    Scrapes the full content from an individual article page.
    Returns the content as a string.
    """
    soup = get_soup(article_url)
    if not soup:
        return 'No Content'

    # Extract Content
    content_div = soup.find('div', class_='entry-content')
    if content_div:
        paragraphs = content_div.find_all('p')
        # Join all paragraphs into a single string
        content = '\n'.join([para.get_text(strip=True) for para in paragraphs])
        return content
    else:
        logging.warning(f"No content found in article {article_url}")
        print(f"Warning: No content found in article {article_url}")
        return 'No Content'

def parse_date(date_text):
    """
    Parses the date string from the category page and returns it in 'YYYY-MM-DD' format.
    """
    try:
        # Expected format: 'November 7, 2024 11:12 AM'
        date = datetime.datetime.strptime(date_text, '%B %d, %Y %I:%M %p')
        formatted_date = date.strftime('%Y-%m-%d')  # Format: 'YYYY-MM-DD'
        return formatted_date
    except ValueError:
        logging.warning(f"Date format unexpected: '{date_text}'")
        print(f"Warning: Date format unexpected: '{date_text}'")
        return 'Unknown'

def scrape_category_page(category_url, target_date):
    """
    Scrapes all articles from a single category page that match the target date.
    Returns a tuple:
        (list of matching articles, bool indicating if scraping should continue)
    """
    soup = get_soup(category_url)
    if not soup:
        return ([], False)

    articles = []

    # Find all article entries
    # Assuming each article is within a <h2 class="entry-title"><a href="...">Title</a></h2>
    article_titles = soup.find_all('h2', class_='entry-title')
    if not article_titles:
        logging.info(f"No articles found on page: {category_url}")
        print(f"No articles found on page: {category_url}")
        return (articles, False)

    # Iterate through each article entry
    for idx, entry in enumerate(article_titles):
        try:
            a_tag = entry.find('a')
            if a_tag and a_tag.has_attr('href'):
                title = a_tag.get_text(strip=True)
                link = a_tag['href']

                # Extract corresponding date
                # Assume the date <p> tag is a sibling or within the same parent
                # Find the next <p> tag with the specific class
                date_p = entry.find_next_sibling('p', class_='mb-0 colorPrimary fs-14')
                if not date_p:
                    # If not found as direct sibling, search within the same parent
                    parent = entry.parent
                    date_p = parent.find('p', class_='mb-0 colorPrimary fs-14')

                if date_p:
                    date_text = date_p.get_text(strip=True)
                    formatted_date = parse_date(date_text)
                    print(f"\nScraping article: {title} | URL: {link}")
                    logging.info(f"Scraping article: {title} | URL: {link}")
                    logging.info(f"Extracted date: {formatted_date}")

                    if formatted_date == target_date:
                        # Scrape full content from article page
                        content = scrape_article_content(link)
                        articles.append({
                            'Title': title,
                            'Link': link,
                            'Date': formatted_date,
                            'Content': content
                        })
                        logging.info(f"Article matches target date: {formatted_date} | Title: {title}")
                        print(f"--> Article matches target date: {formatted_date} | Title: {title}")
                    else:
                        # Date mismatch found; stop scraping
                        logging.info(f"Article skipped (date {formatted_date} does not match target date {target_date}). | Title: {title}")
                        print(f"--> Article skipped (date {formatted_date} does not match target date {target_date}). | Title: {title}")
                        return (articles, False)
                else:
                    logging.warning(f"No date found for article '{title}' on page {category_url}")
                    print(f"Warning: No date found for article '{title}' on page {category_url}")
                    # Depending on policy, you can choose to continue or stop scraping
                    # Here, we'll skip this article and continue
                    continue

                time.sleep(1)  # Polite delay between article requests
            else:
                logging.warning("Article entry without a valid link.")
                print("Warning: Article entry without a valid link.")
        except Exception as e:
            logging.error(f"Error processing article index {idx} on page {category_url}: {e}")
            print(f"Error processing article index {idx} on page {category_url}: {e}")
            continue  # Continue with the next article

    # If all articles on the page matched, continue scraping next page
    return (articles, True)

def scrape_all_pages(base_category_url, target_date):
    """
    Iterates through all paginated category pages and scrapes articles matching the target date.
    Stops paginating when a date mismatch is encountered.
    """
    all_articles = []
    page_number = 1

    while True:
        if page_number == 1:
            page_url = base_category_url
        else:
            page_url = f"{base_category_url}page/{page_number}/"

        logging.info(f"Scraping category page: {page_url}")
        print(f"\nScraping category page: {page_url}")

        articles, should_continue = scrape_category_page(page_url, target_date)
        all_articles.extend(articles)

        if not should_continue:
            logging.info(f"Stopping scraper after processing page {page_number}.")
            print(f"Stopping scraper after processing page {page_number}.")
            break

        page_number += 1
        time.sleep(2)  # Polite delay between category pages

    return all_articles

def save_to_excel(articles, filename):
    """
    Saves the list of articles to an Excel file.
    """
    if not articles:
        logging.warning("No articles to save.")
        print("No articles to save.")
        return

    df = pd.DataFrame(articles)
    try:
        df.to_excel(filename, index=False)
        logging.info(f"Successfully saved {len(df)} articles to {filename}")
        print(f"Successfully saved {len(df)} articles to {filename}")
    except Exception as e:
        logging.error(f"Error saving to Excel: {e}")
        print(f"Error saving to Excel: {e}")

# ------------------------ Main Execution ------------------------

if __name__ == "__main__":
    # -------------------- Target Date Configuration --------------------
    # Allow setting the target date via command-line argument for flexibility
    if len(sys.argv) > 1:
        input_date = sys.argv[1]
        try:
            # Validate the input date format
            datetime.datetime.strptime(input_date, '%Y-%m-%d')
            TARGET_DATE = input_date
        except ValueError:
            logging.error(f"Invalid date format provided: '{input_date}'. Use 'YYYY-MM-DD'.")
            print("Invalid date format. Please use 'YYYY-MM-DD'.")
            sys.exit(1)
    else:
        # Default to the current date
        TARGET_DATE = datetime.date.today().strftime('%Y-%m-%d')  # e.g., '2024-11-07'

    # Base category URL (e.g., National)
    BASE_CATEGORY_URL = 'https://www.newsonair.gov.in/category/national/'  # Modify as needed

    # Output Excel file name
    OUTPUT_FILE = f'air_news_national_{TARGET_DATE}.xlsx'

    # -------------------- Start Scraping --------------------
    start_time = time.time()
    logging.info(f"Starting scraper for date: {TARGET_DATE}")
    print(f"\nStarting scraper for date: {TARGET_DATE}")

    all_scraped_articles = scrape_all_pages(BASE_CATEGORY_URL, TARGET_DATE)
    save_to_excel(all_scraped_articles, OUTPUT_FILE)

    end_time = time.time()
    total_time = end_time - start_time
    logging.info(f"Scraping completed in {total_time:.2f} seconds.")
    print(f"\nScraping completed in {total_time:.2f} seconds.")