Skill: agistack/scraper (version 1.0.0)

--- name: scraper description: Structured extraction and cleanup for public, user-authorized web pages. Use when the user wants to collect, clean, summarize, or transform content from accessible pages into reusable text or data. Do not use to bypass logins, paywalls, captchas, robots restrictions, or access controls. Local-only output. --- # Scraper Turn messy public pages into clean, reusable data. ## Core Purpose Scraper is a safe extraction skill for public, user-authorized pages. It helps the agent: - fetch page content from a URL - extract readable text - strip boilerplate where possible - save clean output locally - prepare content for later summarization or analysis ## Safety Boundaries - Only use on public or user-authorized pages - Do not bypass logins, paywalls, captchas, robots restrictions, or rate limits - Do not request or store credentials - Do not perform stealth scraping, account creation, or identity evasion - Save outputs locally only ## Runtime Requirements - Python 3 must be available as `python3` - No external packages required ## Local Storage All outputs are stored locally under: - `~/.openclaw/workspace/memory/scraper/jobs.json` - `~/.openclaw/workspace/memory/scraper/output/` ## Key Workflows - **Capture a page**: `fetch_page.py --url "https://example.com"` - **Extract readable text**: `extract_text.py --url "https://example.com"` - **Save cleaned content**: `save_output.py --url "https://example.com" --title "Example"` - **List prior jobs**: `list_jobs.py` ## Scripts | Script | Purpose | |---|---| | `init_storage.py` | Initialize scraper storage | | `fetch_page.py` | Download a page with standard headers | | `extract_text.py` | Convert HTML into cleaned plain text | | `save_output.py` | Save extracted output and register a job | | `list_jobs.py` | Show past scraping jobs |

path: references/safety.md
content: # Scraper Safety

This skill is for public, user-authorized web content only.

It must not:
- bypass logins
- bypass paywalls
- bypass captchas
- bypass robots restrictions
- evade rate limits
- collect credentials
- impersonate users or browsers for stealth access

Use it to turn accessible pages into clean local text for downstream analysis.


path: scripts/extract_text.py
content: #!/usr/bin/env python3
import argparse
import re
import urllib.request
from html import unescape

def strip_html(html):
    html = re.sub(r'(?is)<script.*?>.*?</script>', ' ', html)
    html = re.sub(r'(?is)<style.*?>.*?</style>', ' ', html)
    html = re.sub(r'(?i)<br\\s*/?>', '\\n', html)
    html = re.sub(r'(?i)</p>', '\\n', html)
    text = re.sub(r'(?s)<.*?>', ' ', html)
    text = unescape(text)
    text = re.sub(r'\r', '', text)
    text = re.sub(r'\n\s*\n+', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    return text.strip()

def main():
    parser = argparse.ArgumentParser(description="Extract readable text from a public page")
    parser.add_argument("--url", required=True, help="Public URL to fetch and clean")
    args = parser.parse_args()

    req = urllib.request.Request(
        args.url,
        headers={
            "User-Agent": "Mozilla/5.0 (compatible; ScraperSkill/1.0)"
        }
    )

    with urllib.request.urlopen(req, timeout=20) as resp:
        html = resp.read().decode("utf-8", errors="replace")

    text = strip_html(html)
    print(text[:4000])

if __name__ == "__main__":
    main()


path: scripts/fetch_page.py
content: #!/usr/bin/env python3
import argparse
import urllib.request

def main():
    parser = argparse.ArgumentParser(description="Fetch a public web page")
    parser.add_argument("--url", required=True, help="Public URL to fetch")
    args = parser.parse_args()

    req = urllib.request.Request(
        args.url,
        headers={
            "User-Agent": "Mozilla/5.0 (compatible; ScraperSkill/1.0)"
        }
    )

    with urllib.request.urlopen(req, timeout=20) as resp:
        html = resp.read().decode("utf-8", errors="replace")

    print(html[:4000])

if __name__ == "__main__":
    main()


path: scripts/init_storage.py
content: #!/usr/bin/env python3
import json
import os
from datetime import datetime

SCRAPER_DIR = os.path.expanduser("~/.openclaw/workspace/memory/scraper")
OUTPUT_DIR = os.path.join(SCRAPER_DIR, "output")
JOBS_FILE = os.path.join(SCRAPER_DIR, "jobs.json")

def write_json_if_missing(path, payload):
    if not os.path.exists(path):
        with open(path, "w", encoding="utf-8") as f:
            json.dump(payload, f, indent=2, ensure_ascii=False)

def main():
    os.makedirs(SCRAPER_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)

    write_json_if_missing(JOBS_FILE, {
        "metadata": {
            "version": "1.0.0",
            "created_at": datetime.now().isoformat(),
            "last_updated": datetime.now().isoformat()
        },
        "jobs": {}
    })

    print("✓ Scraper storage initialized")
    print(f"  {JOBS_FILE}")
    print(f"  {OUTPUT_DIR}")

if __name__ == "__main__":
    main()


path: scripts/lib/storage.py
content: #!/usr/bin/env python3
import json
import os
from datetime import datetime

SCRAPER_DIR = os.path.expanduser("~/.openclaw/workspace/memory/scraper")
OUTPUT_DIR = os.path.join(SCRAPER_DIR, "output")
JOBS_FILE = os.path.join(SCRAPER_DIR, "jobs.json")

def ensure_dirs():
    os.makedirs(SCRAPER_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)

def load_jobs():
    ensure_dirs()
    if not os.path.exists(JOBS_FILE):
        return {
            "metadata": {
                "version": "1.0.0",
                "created_at": datetime.now().isoformat(),
                "last_updated": datetime.now().isoformat()
            },
            "jobs": {}
        }
    with open(JOBS_FILE, "r", encoding="utf-8") as f:
        return json.load(f)

def save_jobs(data):
    ensure_dirs()
    data.setdefault("metadata", {})
    data["metadata"]["last_updated"] = datetime.now().isoformat()
    tmp = JOBS_FILE + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)
    os.replace(tmp, JOBS_FILE)


path: scripts/list_jobs.py
content: #!/usr/bin/env python3
import os
import sys

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs

def main():
    jobs = load_jobs().get("jobs", {})
    if not jobs:
        print("No scraper jobs saved yet.")
        return

    for job_id, job in jobs.items():
        print(f"{job_id} | {job.get('title')} | {job.get('url')}")
        print(f"  {job.get('saved_to')}")
        print()

if __name__ == "__main__":
    main()


path: scripts/save_output.py
content: #!/usr/bin/env python3
import argparse
import os
import re
import sys
import uuid
from datetime import datetime
import urllib.request
from html import unescape

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs, save_jobs, OUTPUT_DIR

def slugify(value):
    value = value.lower().strip()
    value = re.sub(r'[^a-z0-9]+', '-', value)
    return value.strip('-') or 'page'

def strip_html(html):
    html = re.sub(r'(?is)<script.*?>.*?</script>', ' ', html)
    html = re.sub(r'(?is)<style.*?>.*?</style>', ' ', html)
    html = re.sub(r'(?i)<br\\s*/?>', '\\n', html)
    html = re.sub(r'(?i)</p>', '\\n', html)
    text = re.sub(r'(?s)<.*?>', ' ', html)
    text = unescape(text)
    text = re.sub(r'\r', '', text)
    text = re.sub(r'\n\s*\n+', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    return text.strip()

def main():
    parser = argparse.ArgumentParser(description="Save cleaned output from a public page")
    parser.add_argument("--url", required=True, help="Public URL")
    parser.add_argument("--title", required=True, help="Local title")
    args = parser.parse_args()

    req = urllib.request.Request(
        args.url,
        headers={
            "User-Agent": "Mozilla/5.0 (compatible; ScraperSkill/1.0)"
        }
    )

    with urllib.request.urlopen(req, timeout=20) as resp:
        html = resp.read().decode("utf-8", errors="replace")

    text = strip_html(html)

    job_id = f"JOB-{str(uuid.uuid4())[:6].upper()}"
    filename = f"{slugify(args.title)}.txt"
    output_path = os.path.join(OUTPUT_DIR, filename)

    with open(output_path, "w", encoding="utf-8") as f:
        f.write(text)

    jobs = load_jobs()
    jobs["jobs"][job_id] = {
        "id": job_id,
        "title": args.title,
        "url": args.url,
        "saved_to": output_path,
        "created_at": datetime.now().isoformat()
    }
    save_jobs(jobs)

    print(f"✓ Saved {job_id}")
    print(f"  {output_path}")

if __name__ == "__main__":
    main()


path: skill.json
content: {
  "name": "scraper",
  "version": "1.0.0",
  "description": "Structured extraction and cleanup for public, user-authorized web pages.",
  "entrypoints": [],
  "files": [
    "SKILL.md",
    "skill.json",
    "scripts/init_storage.py",
    "scripts/fetch_page.py",
    "scripts/extract_text.py",
    "scripts/save_output.py",
    "scripts/list_jobs.py",
    "scripts/lib/storage.py",
    "references/safety.md"
  ]
}


Security scan verdicts — ClawScan: clean | static analysis: clean | VirusTotal: clean | SkillSpector: clean (LOW)