1) Project Summary
Build an autonomous web scraper using Playwright that extracts Publix BOGO weekly ad data and exports it to an Excel file.
Goal: Extract Title, Badge, Description, Save Up To, Valid Dates, and Product Image for each BOGO item.
2) Problem Summary
"Save Up To" column always comes out blank.
Selector for "Save Up To" (class .p-price__savings) not consistently available at extraction time.
All other fields extract successfully.
Code times out or fails when waiting for .p-price__savings selector.
3) Things Attempted So Far (bullet list)
- ✅ Initial version extracting Title, Badge, Description, Valid Dates, Image — WORKS.
- ❌ Added extraction for "Save Up To" using .p-price__savings — FAILS, column blank.
- ❌ Global wait_for_selector('.p-price__savings') — FAILS, timeout.
- ❌ Per-card querySelector('.p-price__savings') — FAILS, blank.
- ❌ Extended wait_for_timeout() after page load — FAILS, still blank.
- ✅ Image downloading works perfectly.
- ❌ Tried Playwright EXE packaging — ABANDONED.
4) Ideas to Potentially Fix (bullet list)
- Add longer wait_for_timeout after page load and after each scroll to ensure full rendering.
- Use MutationObserver to watch for dynamic loading of savings elements.
- Try scraping from possible backend API instead of DOM.
- Use Playwright .is_visible() or .inner_text() dynamically rather than querySelector chaining.
5) The Latest Full Code Version (self-contained full code, including all imports)
import asyncio import pandas as pd import xlsxwriter import requests from io import BytesIO from PIL import Image import os from playwright.async_api import async_playwright

PUBLIX_URL = "https://www.publix.com/savings/weekly-ad/bogo" OUTPUT_DIR = "output" os.makedirs(OUTPUT_DIR, exist_ok=True)

def download_image(url, idx): try: response = requests.get(url, timeout=10) response.raise_for_status() img = Image.open(BytesIO(response.content)) img_path = os.path.join(OUTPUT_DIR, f"temp_img_{idx}.png") img.save(img_path) return img_path except: return None

def save_to_excel(items): filename = os.path.join(OUTPUT_DIR, "publix_bogo_deals.xlsx") workbook = xlsxwriter.Workbook(filename) worksheet = workbook.add_worksheet() headers = ["Title", "Badge", "Description", "Save Up To", "Valid Dates"] for col, header in enumerate(headers): worksheet.write(0, col, header) worksheet.set_column("A:E", 35) worksheet.set_column("F:F", 20) for idx, item in enumerate(items): row = idx + 1 worksheet.write(row, 0, item.get("title")) worksheet.write(row, 1, item.get("badge")) worksheet.write(row, 2, item.get("description")) worksheet.write(row, 3, item.get("save_up_to")) worksheet.write(row, 4, item.get("valid")) img_path = download_image(item.get("img"), idx) if img_path: worksheet.set_row(row, 100) worksheet.insert_image(row, 5, img_path, {"x_scale": 0.5, "y_scale": 0.5}) workbook.close()

async def run_scraper(): async with async_playwright() as p: browser = await p.chromium.launch(headless=False) context = await browser.new_context() page = await context.new_page() await page.goto(PUBLIX_URL) await page.wait_for_timeout(8000)

    for _ in range(10):
        await page.mouse.wheel(0, 1500)
        await page.wait_for_timeout(1000)

    await page.wait_for_selector('.p-card', timeout=8000)
    await page.wait_for_timeout(3000)

    extraction_script = """
        () => Array.from(document.querySelectorAll('.p-card')).map(card => ({
            title: card.querySelector('.title-wrapper')?.innerText,
            img: card.querySelector('img')?.src,
            badge: card.querySelector('.p-savings-badge__text')?.innerText,
            description: card.querySelector('.p-savings-badge__description')?.innerText,
            save_up_to: card.querySelector('.p-price__savings')?.innerText || '',
            valid: card.querySelector('.valid-dates')?.innerText
        }))
    """
    items = await page.evaluate(extraction_script)
    print(f"Extracted {len(items)} items.")
    save_to_excel(items)
    await browser.close()
if name == "main": asyncio.run(run_scraper())