Skill: mrherojack/super-lobster (version 1.0.0)

# Super Lobster Use this skill when tasks require aggressive web research, browser rendering, local scripting, crawling, extraction, and command execution on this gateway. ## Operating model - This host is in a China network environment. Do not assume western search engines or API endpoints are reachable or stable. - Prefer first-party sites, direct URLs, mirrors, and browser rendering over public search APIs. - Prefer local execution on the gateway for scraping, parsing, coding, and automation. ## Default workflow 1. If a direct URL is known, fetch it with `fetch_url.py`. 2. If only the readable article body matters, use `extract_main_text.py`. 3. If the page is JS-heavy or renders differently in browsers, use `render_url.py` or `chrome_dump_dom.sh`. 4. If you need to discover more pages inside the same site, use `crawl_site.py`. 5. For multi-step processing, write a Python script under `/root/.openclaw/workspace/memory/tmp` and run it. ## Tools - `/root/.openclaw/workspace/skills/super-lobster/bin/fetch_url.py <url>` Returns metadata plus a text and HTML preview. - `/root/.openclaw/workspace/skills/super-lobster/bin/extract_main_text.py <url>` Extracts the readable main content. - `/root/.openclaw/workspace/skills/super-lobster/bin/chrome_dump_dom.sh <url>` Dumps browser-rendered DOM with Chrome headless. - `/root/.openclaw/workspace/skills/super-lobster/bin/render_url.py <url>` Python wrapper around Chrome headless DOM rendering. - `/root/.openclaw/workspace/skills/super-lobster/bin/crawl_site.py <url> --limit 20` Same-site link discovery crawl. ## Browser rules - Prefer `fetch_url.py` or `extract_main_text.py` first for static pages. - Use browser rendering only when static fetch is incomplete or JS-dependent. - Do not depend on OpenClaw's built-in browser RPC in this environment unless explicitly verified working in the current session. ## Coding and execution rules - You may write and execute Python or shell programs locally on the gateway. - Prefer Python for scraping, parsing, and data cleanup. - Keep scratch outputs in `/root/.openclaw/workspace/memory/tmp`. - Remove clearly temporary files after use unless they are likely to be reused. ## Network rules - Avoid default reliance on DuckDuckGo, Tavily, Exa, or similar public search providers unless reachability is verified first. - If provider keys are added later and connectivity is stable, treat them as accelerators, not hard dependencies.

path: bin/chrome_dump_dom.sh
content: #!/usr/bin/env bash
set -euo pipefail
if [ "$#" -lt 1 ]; then
  echo "usage: $0 <url>" >&2
  exit 2
fi
url="$1"
exec /usr/bin/google-chrome-stable   --headless=new   --disable-gpu   --no-sandbox   --virtual-time-budget=15000   --dump-dom   "$url"


path: bin/crawl_site.py
content: #!/usr/bin/env python3
import argparse
import collections
import json
import sys
from urllib.parse import urljoin, urldefrag, urlparse
import requests
from bs4 import BeautifulSoup

parser = argparse.ArgumentParser()
parser.add_argument('url')
parser.add_argument('--limit', type=int, default=10)
parser.add_argument('--timeout', type=int, default=15)
args = parser.parse_args()

start = args.url
start_host = urlparse(start).netloc
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'}
seen = set()
queue = collections.deque([start])
results = []

while queue and len(results) < args.limit:
    url = queue.popleft()
    if url in seen:
        continue
    seen.add(url)
    try:
        resp = requests.get(url, headers=headers, timeout=args.timeout)
        resp.raise_for_status()
        soup = BeautifulSoup(resp.text, 'lxml')
        title = soup.title.get_text(' ', strip=True) if soup.title else None
        links = []
        for a in soup.select('a[href]'):
            href = a.get('href')
            full = urldefrag(urljoin(resp.url, href))[0]
            if urlparse(full).scheme not in ('http', 'https'):
                continue
            if urlparse(full).netloc != start_host:
                continue
            links.append(full)
        links = list(dict.fromkeys(links))
        results.append({'url': resp.url, 'title': title, 'links': links[:50]})
        for link in links:
            if link not in seen and len(seen) + len(queue) < args.limit * 5:
                queue.append(link)
    except Exception as e:
        results.append({'url': url, 'error': str(e)})

json.dump({'start_url': start, 'results': results}, sys.stdout, ensure_ascii=False, indent=2)
print()


path: bin/extract_main_text.py
content: #!/usr/bin/env python3
import argparse
import json
import sys
import requests
import trafilatura

parser = argparse.ArgumentParser()
parser.add_argument('url')
parser.add_argument('--timeout', type=int, default=20)
args = parser.parse_args()
headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
resp = requests.get(args.url, headers=headers, timeout=args.timeout)
resp.raise_for_status()
html = resp.text
text = trafilatura.extract(html, url=resp.url, include_links=True, include_images=False, favor_precision=True) or ''
result = {
    'url': resp.url,
    'status_code': resp.status_code,
    'title': trafilatura.extract_metadata(html, default_url=resp.url).title if trafilatura.extract_metadata(html, default_url=resp.url) else None,
    'text': text,
}
json.dump(result, sys.stdout, ensure_ascii=False, indent=2)
print()


path: bin/fetch_url.py
content: #!/usr/bin/env python3
import argparse
import json
import sys
import requests
from bs4 import BeautifulSoup

parser = argparse.ArgumentParser()
parser.add_argument('url')
parser.add_argument('--timeout', type=int, default=20)
args = parser.parse_args()

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
resp = requests.get(args.url, headers=headers, timeout=args.timeout)
resp.raise_for_status()
content_type = resp.headers.get('content-type', '')
out = {
    'url': resp.url,
    'status_code': resp.status_code,
    'content_type': content_type,
    'encoding': resp.encoding,
}
if 'html' in content_type:
    soup = BeautifulSoup(resp.text, 'lxml')
    title = soup.title.get_text(' ', strip=True) if soup.title else None
    text = soup.get_text('
', strip=True)
    out['title'] = title
    out['text_preview'] = text[:4000]
    out['html_preview'] = resp.text[:4000]
else:
    out['body_preview'] = resp.text[:4000]
json.dump(out, sys.stdout, ensure_ascii=False, indent=2)
print()


path: bin/render_url.py
content: #!/usr/bin/env python3
import subprocess
import sys

if len(sys.argv) != 2:
    print(f"usage: {sys.argv[0]} <url>", file=sys.stderr)
    sys.exit(2)

url = sys.argv[1]
cmd = [
    "/usr/bin/google-chrome-stable",
    "--headless=new",
    "--disable-gpu",
    "--no-sandbox",
    "--virtual-time-budget=15000",
    "--dump-dom",
    url,
]
proc = subprocess.run(cmd, text=True, capture_output=True)
if proc.returncode != 0:
    sys.stderr.write(proc.stderr)
    sys.exit(proc.returncode)
print(proc.stdout, end="")


Security scan verdicts — ClawScan: suspicious | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (HIGH)