import time
import random
import logging
import pandas as pd
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import (
    NoSuchElementException,
    TimeoutException,
    WebDriverException,
)
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from tqdm import tqdm
from urllib.parse import urljoin

# 1. Configure Logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s - %(levelname)s - %(message)s",
    handlers=[
        logging.FileHandler("scraper.log"),
        logging.StreamHandler(),
    ],
)

# 2. Constants
BASE_URL = "https://www.armedangels.com"
CATEGORY_URL = "https://www.armedangels.com/wo-en/women/tops/t-shirts"

# 3. Selenium Setup
def init_driver(headless=True):
    """
    Initializes the Selenium WebDriver with specified options.
    """
    chrome_options = Options()
    if headless:
        chrome_options.add_argument("--headless")  # Run Chrome in headless mode
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--window-size=1920,1080")
    chrome_options.add_argument("--disable-dev-shm-usage")
    # Optional: Add more options to evade detection
    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option("useAutomationExtension", False)

    try:
        driver = webdriver.Chrome(service=Service("./chromedriver-mac-arm64/chromedriver"), options=chrome_options)
        # Additional script to further evade Selenium detection
        driver.execute_script(
            "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
        )
        logging.info("Chrome WebDriver initialized successfully.")
        return driver
    except WebDriverException as e:
        logging.error(f"Error initializing WebDriver: {e}")
        raise

# 4. Function to Scroll Through the Listings Page
def scroll_to_load_all(driver, pause_time=2, max_scroll_attempts=50):
    """
    Scrolls down the page incrementally to load all products.
    """
    last_height = driver.execute_script("return document.body.scrollHeight")
    scroll_attempt = 0

    while scroll_attempt < max_scroll_attempts:
        # Scroll down to bottom
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

        # Wait to load page
        time.sleep(pause_time)

        # Calculate new scroll height and compare with last scroll height
        new_height = driver.execute_script("return document.body.scrollHeight")
        if new_height == last_height:
            # Check if "No more products" message exists to stop scrolling
            try:
                no_more = driver.find_element(By.XPATH, "//p[contains(text(),'No more products')]")
                if no_more:
                    logging.info("No more products to load.")
                    break
            except NoSuchElementException:
                pass  # Continue if the message is not found
            scroll_attempt += 1
            logging.info(f"No new content loaded. Scroll attempt {scroll_attempt}/{max_scroll_attempts}")
            if scroll_attempt >= max_scroll_attempts:
                logging.warning("Reached maximum scroll attempts. Ending scrolling.")
                break
        else:
            logging.info(f"Loaded new content. Scroll attempt {scroll_attempt}")
            last_height = new_height
            scroll_attempt = 0  # Reset scroll attempt

# 5. Function to Extract Product URLs from the Listings Page
def extract_product_links(driver):
    """
    Parses the current page and extracts all unique product URLs.
    """
    soup = BeautifulSoup(driver.page_source, "html.parser")
    product_links = []
    product_divs = soup.find_all("div", class_="cms-listing-col")

    for div in product_divs:
        a_tag = div.find("a", class_="product-info-link")
        if a_tag and a_tag.get("href"):
            product_url = urljoin(BASE_URL, a_tag["href"])
            product_links.append(product_url)
    unique_links = list(set(product_links))
    logging.info(f"Extracted {len(unique_links)} unique product links.")
    return unique_links

# 6. Function to Scrape Data from Individual Product Pages
def scrape_product_page(driver, url):
    """
    Scrapes detailed information from an individual product page.
    """
    product_data = {
        "SKU": None,
        "Product Name": None,
        "Subline": None,
        "Price": None,
        "Brand": None,
        "GTIN13": None,
        "Release Date": None,
        "Description": None,
        "Image URLs": None,
        "Available Sizes": None,
        "Colors": None,
        "Materials": None,
        "Certifications": None,
        "Fit & Size": None,
        "Shipping & Returns": None,
        "Product URL": url,
    }

    try:
        driver.get(url)
        # Wait until the product name is loaded to ensure the page is fully loaded
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "h1.product-detail-name"))
        )
        # Additional wait for dynamic content
        time.sleep(random.uniform(1, 2))

        soup = BeautifulSoup(driver.page_source, "html.parser")

        # 6.1. Extract SKU
        sku_tag = soup.find("meta", {"itemprop": "sku"})
        if sku_tag and sku_tag.has_attr("content"):
            product_data["SKU"] = sku_tag["content"]
        else:
            logging.warning(f"SKU not found for {url}")

        # 6.2. Extract Product Name and Subline
        name_tag = soup.find("h1", class_="product-detail-name")
        if name_tag:
            headline = name_tag.find("div", class_="headline")
            subline = name_tag.find("div", class_="subline")
            if headline:
                product_data["Product Name"] = headline.get_text(strip=True)
            else:
                logging.warning(f"Headline not found for {url}")

            if subline:
                product_data["Subline"] = subline.get_text(strip=True)
            else:
                logging.warning(f"Subline not found for {url}")
        else:
            logging.warning(f"Product name not found for {url}")

        # 6.3. Extract Price
        price_tag = soup.find("p", class_="product-price")
        if price_tag:
            product_data["Price"] = price_tag.get_text(strip=True)
        else:
            logging.warning(f"Price not found for {url}")

        # 6.4. Extract Brand
        brand_meta = soup.find("meta", {"itemprop": "name"})
        if brand_meta and brand_meta.has_attr("content"):
            product_data["Brand"] = brand_meta["content"]
        else:
            logging.warning(f"Brand not found for {url}")

        # 6.5. Extract GTIN13
        gtin = soup.find("meta", {"itemprop": "gtin13"})
        if gtin and gtin.has_attr("content"):
            product_data["GTIN13"] = gtin["content"]
        else:
            logging.warning(f"GTIN13 not found for {url}")

        # 6.6. Extract Release Date
        release_date = soup.find("meta", {"itemprop": "releaseDate"})
        if release_date and release_date.has_attr("content"):
            product_data["Release Date"] = release_date["content"]
        else:
            logging.warning(f"Release Date not found for {url}")

        # 6.7. Extract Description
        description_tag = soup.find("p", itemprop="description")
        if description_tag:
            product_data["Description"] = description_tag.get_text(strip=True)
        else:
            # Alternative selector based on provided HTML snippet
            description_div = soup.find("div", class_="card-body body-text mb-0")
            if description_div:
                description_p = description_div.find("p", class_="body-text")
                if description_p:
                    product_data["Description"] = description_p.get_text(strip=True)
                else:
                    # Handle possible paragraphs directly within the div
                    direct_p = description_div.find("p", recursive=False)
                    if direct_p:
                        product_data["Description"] = direct_p.get_text(strip=True)
                    else:
                        logging.warning(f"Description paragraph not found in card-body for {url}")
            else:
                logging.warning(f"Description not found for {url}")

        # 6.8. Extract Image URLs
        images = []
        image_tags = soup.find_all("img", class_="gallery-slider-image")
        if image_tags:
            for img in image_tags:
                full_image = img.get("data-full-image")
                if full_image:
                    full_image_url = urljoin(BASE_URL, full_image)
                    images.append(full_image_url)
                else:
                    # Fallback to 'src' if 'data-full-image' is not available
                    src = img.get("src")
                    if src:
                        images.append(urljoin(BASE_URL, src))
        else:
            logging.warning(f"No images found for {url}")
        product_data["Image URLs"] = "; ".join(images) if images else None

        # 6.9. Extract Available Sizes
        sizes = []
        size_labels = soup.find_all("label", class_="swatch__size-item")
        if size_labels:
            for label in size_labels:
                size_text = label.get_text(strip=True)
                if size_text:
                    sizes.append(size_text)
        else:
            logging.warning(f"No size labels found for {url}")
        product_data["Available Sizes"] = ", ".join(sizes) if sizes else None

        # 6.10. Extract Colors
        colors = []
        color_sections = soup.find_all("a", class_="swatch__color")
        if color_sections:
            for color in color_sections:
                color_title = color.get("title", "").strip()
                color_style = color.find("span", class_="swatch__color-item")
                if color_style and color_style.get("style"):
                    try:
                        color_code = color_style["style"].split(":")[1].strip().rstrip(";")
                        colors.append(f"{color_title} ({color_code})")
                    except IndexError:
                        colors.append(color_title)
                        logging.warning(f"Could not parse color code for {url}")
                else:
                    if color_title:
                        colors.append(f"{color_title}")
                        logging.warning(f"No color code found for {url}")
        else:
            logging.warning(f"No color sections found for {url}")
        product_data["Colors"] = "; ".join(colors) if colors else None

        # 6.11. Extract Materials
        materials = []
        li_tags = soup.find_all("li")
        for li in li_tags:
            text = li.get_text(strip=True)
            if "Main Fabric:" in text:
                span = li.find("span")
                material_text = span.get_text(strip=True) if span else None
                if material_text:
                    materials.append(material_text)
                else:
                    logging.warning(f"Main Fabric span not found for {url}")
        product_data["Materials"] = ", ".join(materials) if materials else None

        # 6.12. Extract Certifications
        certifications = []
        certification_divs = soup.find_all("div", class_="certification")
        if certification_divs:
            for cert in certification_divs:
                cert_text_tag = cert.find("p")
                if cert_text_tag:
                    cert_text = cert_text_tag.get_text(separator=" ", strip=True)
                    certifications.append(cert_text)
                else:
                    logging.warning(f"Certification text not found for {url}")
        else:
            logging.warning(f"No certifications found for {url}")
        product_data["Certifications"] = "; ".join(certifications) if certifications else None

        # 6.13. Extract Fit & Size Details
        fit_and_size = {}
        # Find all headings related to Fit & Size
        fit_headings = soup.find_all("p", class_="product-detail-accordion-heading")
        for heading in fit_headings:
            section_title = heading.get_text(strip=True)
            if section_title in ["Fit", "Measures"]:
                # Get the next sibling element which usually contains the data
                section_content = heading.find_next_sibling(["ul", "div"])
                if section_content:
                    items = {}
                    if section_content.name == "ul":
                        for li in section_content.find_all("li"):
                            # Split by colon
                            if li.find("span"):
                                key = li.find("span").previous_sibling
                                key = key.strip(": ").strip() if key else None
                                value = li.find("span").get_text(strip=True)
                                if key and value:
                                    items[key] = value
                            else:
                                # Handle nested lists (e.g., under "Modelsize")
                                nested_ul = li.find("ul")
                                if nested_ul:
                                    for nested_li in nested_ul.find_all("li"):
                                        nested_text = nested_li.get_text(strip=True)
                                        if nested_text:
                                            items.setdefault("Modelsize", []).append(nested_text)
                    elif section_content.name == "div":
                        # Handle other possible structures if necessary
                        pass
                    if items:
                        fit_and_size[section_title] = items
        if fit_and_size:
            product_data["Fit & Size"] = fit_and_size
        else:
            logging.warning(f"Fit & Size details not found for {url}")

        # 6.14. Extract Shipping & Returns
        shipping = soup.find("div", class_="shipping-text")
        if shipping:
            product_data["Shipping & Returns"] = shipping.get_text(separator=" ", strip=True)
        else:
            logging.warning(f"Shipping & Returns info not found for {url}")

        return product_data

# 7. Function to Save Data to CSV
def save_to_csv(data, filename):
    """
    Converts the list of product dictionaries into a DataFrame and saves it as a CSV file.
    """
    if not data:
        logging.info("No data to save.")
        return

    df = pd.DataFrame(data)
    # Expand 'Fit & Size' dictionary into separate columns if it exists
    if "Fit & Size" in df.columns:
        # Create a DataFrame from the 'Fit & Size' column
        fit_size_df = df["Fit & Size"].apply(
            lambda x: pd.Series(x) if isinstance(x, dict) else pd.Series()
        )
        # Expand nested dictionaries within 'Fit & Size'
        for column in fit_size_df.columns:
            nested_dict = fit_size_df[column].dropna()
            if isinstance(nested_dict.iloc[0], dict):
                nested_df = nested_dict.apply(pd.Series)
                nested_df.columns = [f"{column} - {subcol}" for subcol in nested_df.columns]
                fit_size_df = pd.concat(
                    [fit_size_df.drop([column], axis=1), nested_df], axis=1
                )
            elif isinstance(nested_dict.iloc[0], list):
                # Join list items with a comma
                fit_size_df[column] = fit_size_df[column].apply(
                    lambda x: ", ".join(x) if isinstance(x, list) else x
                )
        # Concatenate the expanded 'Fit & Size' DataFrame with the main DataFrame
        df = pd.concat([df.drop(["Fit & Size"], axis=1), fit_size_df], axis=1)

    # Save to CSV
    df.to_csv(filename, index=False, encoding="utf-8-sig")
    logging.info(f"Data saved to {filename}")

# 8. Main Scraping Function
def main():
    """
    Main function to orchestrate the scraping process.
    """
    driver = None
    try:
        driver = init_driver(headless=True)

        # Navigate to the category page
        driver.get(CATEGORY_URL)
        logging.info(f"Navigated to {CATEGORY_URL}")

        # Scroll through the page to load all products
        scroll_to_load_all(driver)
        logging.info("Completed scrolling through the listings page.")

        # Extract product links
        product_links = extract_product_links(driver)
        logging.info(f"Extracted {len(product_links)} unique product links.")

        if not product_links:
            logging.error("No product links found. Exiting.")
            return

        # Scrape each product page
        all_product_data = []
        logging.info("Starting to scrape individual product pages:")
        for link in tqdm(product_links, desc="Scraping Products"):
            product_info = scrape_product_page(driver, link)
            if product_info:
                all_product_data.append(product_info)
            # Polite scraping: wait between 1 to 3 seconds
            time.sleep(random.uniform(1, 3))

        if not all_product_data:
            logging.error("No product data scraped. Exiting.")
            return

        # Save data to CSV
        output_filename = "armedangels_tshirts_detailed.csv"
        save_to_csv(all_product_data, output_filename)

    except WebDriverException as e:
        logging.error(f"WebDriverException occurred: {e}")
    except Exception as e:
        logging.error(f"An unexpected error occurred: {e}")
    finally:
        if driver:
            driver.quit()
            logging.info("Chrome WebDriver closed.")

# 9. Run the Scraper
if __name__ == "__main__":
    main()

Fix code -