Skill: agistack/fetch (version 1.0.0)

--- name: fetch description: Public web retrieval and clean extraction engine. Use whenever the user wants to fetch, download, inspect, clean, or save content from a public URL. Supports safe fetching of public web pages, extracts title/text/links, stores results locally, and keeps a job history. No credentials, no login flows, no cloud sync. --- # Fetch Turn public URLs into usable local content. ## Core Philosophy 1. Fetch only public web content. 2. Prefer clean extracted text over noisy raw HTML. 3. Save both the raw response and structured extraction locally. 4. Keep a simple local job history so previous fetches are easy to inspect. ## Runtime Requirements - Python 3 must be available as `python3` - No external packages required ## Safety Boundaries - Public URLs only - No login flows - No cookies or browser automation - No API keys or credentials - No external uploads or cloud sync - All fetched data is stored locally only ## Local Storage All data is stored under: - `~/.openclaw/workspace/memory/fetch/jobs.json` - `~/.openclaw/workspace/memory/fetch/pages/` ## Key Workflows - **Fetch URL**: `fetch_url.py --url "https://example.com"` - **Save cleaned output**: `save_output.py --url "https://example.com" --title "Example"` - **List history**: `list_jobs.py` - **Show job details**: `show_job.py --id JOB-XXXX` ## Scripts | Script | Purpose | |---|---| | `init_storage.py` | Initialize local storage | | `fetch_url.py` | Fetch a public URL and extract content | | `save_output.py` | Save cleaned output with a custom title | | `list_jobs.py` | List previous fetch jobs | | `show_job.py` | Show one saved fetch job |

path: references/philosophy.md
content: # Fetch Philosophy

1. Fetch only public content.
2. Keep the output useful and readable.
3. Save raw + cleaned results locally.
4. Minimize surprise, maximize traceability.


path: scripts/fetch_url.py
content: #!/usr/bin/env python3
import argparse
import json
import os
import sys
import uuid
from datetime import datetime
from urllib.parse import urlparse
from urllib.request import Request, urlopen

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs, save_jobs, PAGES_DIR
from lib.extract import strip_html, extract_title, extract_links

def main():
    parser = argparse.ArgumentParser(description="Fetch a public URL")
    parser.add_argument("--url", required=True, help="Public URL to fetch")
    args = parser.parse_args()

    parsed = urlparse(args.url)
    if parsed.scheme not in ("http", "https"):
        print("Only http/https URLs are allowed.")
        sys.exit(1)

    req = Request(
        args.url,
        headers={"User-Agent": "Mozilla/5.0 (compatible; FetchSkill/1.0.0)"}
    )

    with urlopen(req, timeout=20) as resp:
        raw_bytes = resp.read()
        content_type = resp.headers.get("Content-Type", "")
        status_code = getattr(resp, "status", 200)

    raw_html = raw_bytes.decode("utf-8", errors="replace")
    title = extract_title(raw_html)
    clean_text = strip_html(raw_html)
    links = extract_links(raw_html)

    job_id = f"JOB-{str(uuid.uuid4())[:4].upper()}"
    now = datetime.now().isoformat()

    raw_path = os.path.join(PAGES_DIR, f"{job_id}.raw.html")
    clean_path = os.path.join(PAGES_DIR, f"{job_id}.clean.txt")

    with open(raw_path, "w", encoding="utf-8") as f:
        f.write(raw_html)
    with open(clean_path, "w", encoding="utf-8") as f:
        f.write(clean_text)

    data = load_jobs()
    data["jobs"][job_id] = {
        "id": job_id,
        "url": args.url,
        "title": title,
        "status_code": status_code,
        "content_type": content_type,
        "created_at": now,
        "raw_path": raw_path,
        "clean_path": clean_path,
        "text_chars": len(clean_text),
        "link_count": len(links)
    }
    save_jobs(data)

    print(f"✓ Fetched: {job_id}")
    print(f"  Title: {title}")
    print(f"  Status: {status_code}")
    print(f"  Links: {len(links)}")
    print(f"  Clean text chars: {len(clean_text)}")

if __name__ == "__main__":
    main()


path: scripts/init_storage.py
content: #!/usr/bin/env python3
import json
import os
from datetime import datetime

FETCH_DIR = os.path.expanduser("~/.openclaw/workspace/memory/fetch")
PAGES_DIR = os.path.join(FETCH_DIR, "pages")
JOBS_FILE = os.path.join(FETCH_DIR, "jobs.json")

def write_json_if_missing(path, payload):
    if not os.path.exists(path):
        with open(path, "w", encoding="utf-8") as f:
            json.dump(payload, f, indent=2, ensure_ascii=False)

def main():
    os.makedirs(FETCH_DIR, exist_ok=True)
    os.makedirs(PAGES_DIR, exist_ok=True)

    write_json_if_missing(JOBS_FILE, {
        "metadata": {
            "version": "1.0.0",
            "created_at": datetime.now().isoformat(),
            "last_updated": datetime.now().isoformat()
        },
        "jobs": {}
    })

    print("✓ Fetch storage initialized")
    print(f"  {JOBS_FILE}")
    print(f"  {PAGES_DIR}")

if __name__ == "__main__":
    main()


path: scripts/lib/extract.py
content: #!/usr/bin/env python3
import html
import re
from html.parser import HTMLParser

class LinkParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.links = []

    def handle_starttag(self, tag, attrs):
        if tag.lower() == "a":
            href = dict(attrs).get("href")
            if href:
                self.links.append(href)

def strip_html(raw_html: str) -> str:
    text = re.sub(r"(?is)<script.*?>.*?</script>", " ", raw_html)
    text = re.sub(r"(?is)<style.*?>.*?</style>", " ", text)
    text = re.sub(r"(?s)<[^>]+>", " ", text)
    text = html.unescape(text)
    text = re.sub(r"\r", "", text)
    text = re.sub(r"\n\s*\n+", "\n\n", text)
    text = re.sub(r"[ \t]+", " ", text)
    return text.strip()

def extract_title(raw_html: str) -> str:
    m = re.search(r"(?is)<title>(.*?)</title>", raw_html)
    if not m:
        return "Untitled"
    return html.unescape(m.group(1)).strip()

def extract_links(raw_html: str):
    parser = LinkParser()
    parser.feed(raw_html)
    return parser.links[:100]


path: scripts/lib/storage.py
content: #!/usr/bin/env python3
import json
import os
from datetime import datetime

FETCH_DIR = os.path.expanduser("~/.openclaw/workspace/memory/fetch")
PAGES_DIR = os.path.join(FETCH_DIR, "pages")
JOBS_FILE = os.path.join(FETCH_DIR, "jobs.json")

def ensure_dirs():
    os.makedirs(FETCH_DIR, exist_ok=True)
    os.makedirs(PAGES_DIR, exist_ok=True)

def _safe_load(path, default):
    ensure_dirs()
    if not os.path.exists(path):
        return default
    try:
        with open(path, "r", encoding="utf-8") as f:
            return json.load(f)
    except (json.JSONDecodeError, OSError):
        return default

def _atomic_save(path, data):
    ensure_dirs()
    tmp = path + ".tmp"
    with open(tmp, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)
    os.replace(tmp, path)

def load_jobs():
    return _safe_load(JOBS_FILE, {
        "metadata": {
            "version": "1.0.0",
            "created_at": datetime.now().isoformat(),
            "last_updated": datetime.now().isoformat()
        },
        "jobs": {}
    })

def save_jobs(data):
    data.setdefault("metadata", {})
    data["metadata"]["last_updated"] = datetime.now().isoformat()
    _atomic_save(JOBS_FILE, data)


path: scripts/list_jobs.py
content: #!/usr/bin/env python3
import os
import sys

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs

def main():
    jobs = load_jobs().get("jobs", {})
    if not jobs:
        print("No fetch jobs yet.")
        return

    for job in sorted(jobs.values(), key=lambda x: x["created_at"], reverse=True):
        print(f"{job['id']} | {job['title']} | {job['url']}")

if __name__ == "__main__":
    main()


path: scripts/save_output.py
content: #!/usr/bin/env python3
import argparse
import os
import sys
from datetime import datetime

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs, PAGES_DIR

def slugify(text: str) -> str:
    safe = "".join(c.lower() if c.isalnum() else "-" for c in text)
    while "--" in safe:
        safe = safe.replace("--", "-")
    return safe.strip("-") or "output"

def main():
    parser = argparse.ArgumentParser(description="Save cleaned output with a custom title")
    parser.add_argument("--url", required=True, help="URL to find in job history")
    parser.add_argument("--title", required=True, help="Output title")
    args = parser.parse_args()

    jobs = load_jobs().get("jobs", {})
    matches = [j for j in jobs.values() if j.get("url") == args.url]
    if not matches:
        print("No fetched job found for this URL.")
        sys.exit(1)

    latest = sorted(matches, key=lambda x: x["created_at"], reverse=True)[0]
    clean_path = latest["clean_path"]

    with open(clean_path, "r", encoding="utf-8") as f:
        content = f.read()

    filename = f"{slugify(args.title)}-{datetime.now().strftime('%Y%m%d-%H%M%S')}.txt"
    out_path = os.path.join(PAGES_DIR, filename)

    with open(out_path, "w", encoding="utf-8") as f:
        f.write(content)

    print("✓ Saved cleaned output")
    print(f"  {out_path}")

if __name__ == "__main__":
    main()


path: scripts/show_job.py
content: #!/usr/bin/env python3
import argparse
import json
import os
import sys

SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, SCRIPT_DIR)

from lib.storage import load_jobs

def main():
    parser = argparse.ArgumentParser(description="Show one fetch job")
    parser.add_argument("--id", required=True, help="Job ID")
    args = parser.parse_args()

    jobs = load_jobs().get("jobs", {})
    if args.id not in jobs:
        print(f"Job not found: {args.id}")
        sys.exit(1)

    print(json.dumps(jobs[args.id], indent=2, ensure_ascii=False))

if __name__ == "__main__":
    main()


path: skill.json
content: {
  "name": "fetch",
  "version": "1.0.0",
  "description": "Public web retrieval and clean extraction engine for public URLs.",
  "entrypoints": [],
  "files": [
    "SKILL.md",
    "skill.json",
    "scripts/init_storage.py",
    "scripts/fetch_url.py",
    "scripts/save_output.py",
    "scripts/list_jobs.py",
    "scripts/show_job.py",
    "scripts/lib/storage.py",
    "scripts/lib/extract.py",
    "references/philosophy.md"
  ]
}


Security scan verdicts — ClawScan: clean | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (MEDIUM)