🔍  Query: generative interpretation
Solve CAPTCHA in the window…
Traceback (most recent call last):
  File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 183, in <module>
    asyncio.run(main())
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 171, in main
    res = await s.search(args.query, args.max)
  File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 153, in search
    results = await self._parse_results(page, k)
  File "/Users/yarbel/bbgpt/utils/bbgpt/gschol2.py", line 110, in _parse_results
    await page.wait_for_selector("div.gs_r", timeout=15_000)
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/async_api/_generated.py", line 8180, in wait_for_selector
    await self._impl_obj.wait_for_selector(
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_page.py", line 425, in wait_for_selector
    return await self._main_frame.wait_for_selector(**locals_to_params(locals()))
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_frame.py", line 323, in wait_for_selector
    await self._channel.send("waitForSelector", locals_to_params(locals()))
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_connection.py", line 61, in send
    return await self._connection.wrap_api_call(
  File "/Users/yarbel/miniconda3/envs/gen/lib/python3.10/site-packages/playwright/_impl/_connection.py", line 528, in wrap_api_call
    raise rewrite_error(error, f"{parsed_st['apiName']}: {error}") from None
playwright._impl._errors.TargetClosedError: Page.wait_for_selector: Target page, context or browser has been closed
(gen) yarbel@UAs-MacBook-Pro bbgpt % 

# ── OVERWRITE gschol2.py ──
cat > gschol2.py << 'EOF'
"""
gschol2.py  ·  Headless-by-default Google Scholar scraper with one-time
CAPTCHA fallback (WebKit by default, Chromium/Firefox optional).

Key points
──────────
• Runs headless. If Scholar blocks with the “prove you’re not a robot”
  page (including Hebrew “הוכח שאינך רובוט”), it automatically pops a
  visible WebKit window so you can tick the box.
• After you solve once, cookies are saved to --user-data-dir
  (default: ~/.scholar_state.json). Subsequent headless runs reuse them,
  so you shouldn’t see the CAPTCHA again.
• Clicks the dynamic “Cite” link and scrapes MLA/Chicago rows to grab
  full author names (flagged with ✅).

Dependencies   →  uv pip install playwright playwright-stealth rich
Playwright     →  playwright install chromium webkit firefox
"""

import asyncio, random, re, json, pathlib, os
from typing import List, Dict, Any, Optional
from rich.console import Console
from playwright.async_api import async_playwright, Page
from playwright_stealth import stealth_async

console = Console()

# ╭──────────── helpers ────────────╮
async def snooze(a=0.5, b=2.0):  # human-ish pauses
    await asyncio.sleep(random.uniform(a, b))

def is_captcha(html: str) -> bool:
    return ("gs_captcha_f" in html or
            "prove you are not a robot" in html.lower() or
            "הוכח שאינך רובוט" in html)

def file_exists(path: Optional[str]) -> bool:
    return bool(path) and pathlib.Path(path).expanduser().exists()

# ╭──────────── citation extractor ────────────╮
async def full_authors_from_modal(page: Page) -> Optional[str]:
    rows = await page.query_selector_all("tr.gs_citi_tr")
    for row in rows:
        lbl = await row.query_selector("th.gs_citi_lbl")
        if lbl and ("MLA" in await lbl.inner_text() or "Chicago" in await lbl.inner_text()):
            txt = await (await row.query_selector("div.gs_citr")).inner_text()
            m = re.match(r'^([^"\.(]+?)(?:\.|")', txt.strip())
            if m and re.search(r'[A-Z][a-z]{2,}\s+[A-Z][a-z]{2,}', m[1]):
                return m[1].strip()
    return None

# ╭──────────── scraper class ────────────╮
class ScholarScraper:
    def __init__(self, engine: str = "webkit", state_file: str = "~/.scholar_state.json"):
        self.engine_name = engine
        self.state_path = str(pathlib.Path(state_file).expanduser())

    async def _launch(self, p, headless=True):
        launcher = {"chromium": p.chromium,
                    "firefox": p.firefox,
                    "webkit":  p.webkit}[self.engine_name]
        browser = await launcher.launch(headless=headless,
                                        args=["--disable-blink-features=AutomationControlled",
                                              "--disable-infobars",
                                              "--window-size=1920,1080"])
        ctx_kwargs = {
            "viewport": {"width": 1920, "height": 1080},
            # generic UA (Scholar doesn’t care which engine)
            "user_agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                           "AppleWebKit/537.36 (KHTML, like Gecko) "
                           "Chrome/120.0.0.0 Safari/537.36")
        }
        # load cookies if we have them
        if file_exists(self.state_path):
            ctx_kwargs["storage_state"] = self.state_path
        context = await browser.new_context(**ctx_kwargs)
        await context.add_init_script(
            """
            Object.defineProperty(navigator,'webdriver',{get:()=>undefined});
            Object.defineProperty(navigator,'plugins',{get:()=>[1,2,3]});
            Object.defineProperty(navigator,'languages',{get:()=>['en-US','en']});
            const gp=WebGLRenderingContext.prototype.getParameter;
            WebGLRenderingContext.prototype.getParameter=function(p){
              if(p===37445) return 'Intel Inc.';
              if(p===37446) return 'Intel Iris OpenGL Engine';
              return gp.call(this,p);
            };
            """)
        page = await context.new_page()
        await stealth_async(page)
        return browser, context, page

    # manual CAPTCHA path (always WebKit headful to minimise risk)
    async def _solve_captcha(self, query: str):
        async with async_playwright() as p:
            browser, ctx, page = await self._launch(p, headless=False)
            await page.goto("https://scholar.google.com", wait_until="domcontentloaded")
            console.print("[yellow]Solve CAPTCHA in the window…[/yellow]")
            await page.wait_for_selector("input#gs_hdr_tsi", timeout=120_000)
            box = await page.query_selector("input#gs_hdr_tsi")
            await box.fill(query); await box.press("Enter")
            await page.wait_for_selector("div.gs_r", timeout=20_000)
            # save cookies
            pathlib.Path(self.state_path).parent.mkdir(parents=True, exist_ok=True)
            await ctx.storage_state(path=self.state_path)
            # do not close context—parse results directly
            return ctx, page

    async def _parse_results(self, page: Page, k: int) -> List[Dict[str, Any]]:
        await page.wait_for_selector("div.gs_r", timeout=15_000)
        entries = (await page.query_selector_all("div.gs_r"))[:k]
        out=[]
        for e in entries:
            t_el = await e.query_selector("h3.gs_rt")
            title = (await t_el.inner_text()).replace("[PDF]","").replace("[HTML]","").strip()
            url = await (await t_el.query_selector("a")).get_attribute("href") if t_el else ""
            meta = await (await e.query_selector("div.gs_a")).inner_text()
            authors_orig = meta.split(" - ")[0]
            year_match = re.search(r"\b(19|20)\d{2}\b", meta)
            year = year_match.group() if year_match else "N/A"
            cites = 0
            cb = await e.query_selector("a:has-text('Cited by')")
            if cb:
                m = re.search(r"Cited by (\d+)", await cb.inner_text())
                cites = int(m[1]) if m else 0
            cite_btn = await e.query_selector("a.gs_or_cit")
            full=None
            if cite_btn:
                await snooze(0.8,2.0); await cite_btn.click()
                await page.wait_for_selector("div.gs_citi", timeout=8_000)
                full = await full_authors_from_modal(page)
                await page.keyboard.press("Escape")
            out.append({
                "title": title,
                "url": url,
                "authors": full or authors_orig,
                "full_names": bool(full),
                "year": year,
                "citations": cites,
            })
        return out

    async def search(self, query: str, k: int = 5) -> List[Dict[str, Any]]:
        async with async_playwright() as p:
            # first try headless
            browser, ctx, page = await self._launch(p, headless=True)
            await page.goto(f"https://scholar.google.com/scholar?q={query}&hl=en&num={k}",
                            wait_until="domcontentloaded")
            if is_captcha(await page.content()):
                await ctx.close(); await browser.close()
                # fallback headful solve
                ctx, page = await self._solve_captcha(query)
                results = await self._parse_results(page, k)
                await ctx.close()
                return results
            results = await self._parse_results(page, k)
            await ctx.close(); await browser.close()
            return results

# ╭──────────── CLI ────────────╮
async def main():
    import argparse
    ap = argparse.ArgumentParser(description="Scholar scraper headless → headful fallback")
    ap.add_argument("query"); ap.add_argument("--max", type=int, default=5)
    ap.add_argument("--browser", choices=["webkit","chromium","firefox"], default="webkit")
    ap.add_argument("--user-data-dir", default="~/.scholar_state.json",
                    help="where to cache cookies (JSON)")
    args = ap.parse_args()
    console.print(f"🔍  [green]Query:[/green] {args.query}")
    s = ScholarScraper(engine=args.browser, state_file=args.user_data_dir)
    res = await s.search(args.query, args.max)
    if not res:
        console.print("[red]No results.[/red]"); return
    for i,r in enumerate(res,1):
        tick="✅" if r["full_names"] else ""
        console.print(f"{i}. [magenta]{r['title']}[/magenta] {tick}")
        console.print(f"   • {r['authors']} ({r['year']}) — cites: {r['citations']}\n")
    # save
    with open("scholar_results.json","w",encoding="utf-8") as f: json.dump(res,f,indent=2,ensure_ascii=False)
    console.print("[cyan]Saved to scholar_results.json[/cyan]")

if __name__ == "__main__":
    asyncio.run(main())
EOF

# ── immediate test run ──
python gschol2.py "generative interpretation" --max 3 --browser webkit --user-data-dir ~/.scholar_state.json

Find a better implementation, one that will slay the captcha nonsense, and will allow us to run this in headless while being able to click the cite button to get bibtext information. Write it in cat > file format.