Skill: youjunzhao/awesome-paper-skill (version 1.0.2)

--- name: awesome-paper-skill description: End-to-end pipeline for topic-driven literature research: collect papers from multiple sources, generate an Awesome-style README, and update/push to user GitHub repo. --- # Awesome Paper Skill Use this skill when user asks for a full pipeline: 1. User gives a **topic**. 2. Agent researches papers (multi-source, not arXiv-only). 3. Agent builds/updates an Awesome-style `README.md`. 4. Agent pushes changes to user GitHub repo. ## Required Inputs - `topic` (required) - `repo_owner` (required) - `repo_name` (required) - `visibility` (optional, default: keep current repo visibility) - `max_arxiv` / `max_crossref` / `max_semantic` (optional) If owner/repo are omitted, use defaults from current workspace context or ask once. ## Pipeline ### 1) Fetch papers (multi-source) ```bash python3 skills/awesome-paper-skill/scripts/fetch_papers.py \ --topic "<topic>" \ --max-arxiv 60 \ --max-crossref 60 \ --max-semantic 60 \ --out /tmp/research_papers.json ``` Policy: - Merge arXiv + Crossref + Semantic Scholar. - Continue on partial source failures; report coverage gaps. - De-duplicate by title/doi. ### 2) Build Awesome README ```bash python3 skills/awesome-paper-skill/scripts/build_awesome_readme.py \ --topic "<topic>" \ --input /tmp/research_papers.json \ --output /tmp/README.md ``` Formatting rules (strict): - English-only. - One paper per bullet. - If venue known: show `[Venue]` line. - If venue unknown: do **not** print `[Preprint]` line. - Keep arXiv badge line. - Keep GitHub badge line only when valid repo exists. - Do **not** include Website placeholders/badges. - Date on its own line: `(YYYY-MM-DD)`. ### 3) Publish/update GitHub repo ```bash python3 skills/awesome-paper-skill/scripts/publish_repo.py \ --owner "<repo_owner>" \ --name "<repo_name>" \ --readme /tmp/README.md \ --visibility private ``` If repo exists: update README and push. If repo does not exist: create then push. ## Quality Gates Before final reply: - README renders. - `Total papers` count matches entries. - No Website placeholder links. - No fake `Repo Not Found` badges. - Repo push succeeded. ## Deliverables to user - Brief summary (what changed) - GitHub commit/repo URL

path: _meta.json
content: ownerId: kn72thwj42fqymrwn8pas9fwk1833phy
slug: awesome-paper-skill
version: 1.0.0
publishedAt: 1773824179912

path: scripts/build_awesome_readme.py
content: #!/usr/bin/env python3
import argparse
import json
import re
from collections import defaultdict
from datetime import datetime, timezone


AWESOME_BADGE = "[![Awesome](https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg)](https://github.com/sindresorhus/awesome)"


def topic_bucket(text: str):
    t = (text or "").lower()
    if any(k in t for k in ["security", "attack", "defense", "safety", "guardrail", "prompt injection"]):
        return "Security and Trustworthy Agents"
    if any(k in t for k in ["agent", "system", "architecture", "resource", "routing", "os"]):
        return "System Architecture and Resource Management"
    if any(k in t for k in ["memory", "benchmark", "evaluation", "reinforcement", "rl", "learning"]):
        return "Memory, Learning, and Evaluation"
    return "Applications and Human Factors"


def has_published_venue(p):
    return bool((p.get("venue") or "").strip())


def badge(label, link, color):
    return f"[![{label}](https://img.shields.io/badge/{label}-{color}.svg)]({link})"


def venue_tag(p):
    venue = (p.get("venue") or "").strip()
    if not venue:
        return ""
    clean = re.sub(r"\s+", " ", venue)
    return f"[{clean}]"


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--topic", required=True)
    ap.add_argument("--input", required=True)
    ap.add_argument("--output", required=True)
    args = ap.parse_args()

    with open(args.input, "r", encoding="utf-8") as f:
        data = json.load(f)

    papers = data.get("papers", [])

    grouped = defaultdict(list)
    for p in papers:
        primary_text = f"{p.get('title', '')} {p.get('abstract', '')}"
        tcat = topic_bucket(primary_text)
        grouped[tcat].append(p)

    lines = []
    lines.append(f"# Awesome {args.topic.title()} Papers {AWESOME_BADGE}")
    lines.append("")
    lines.append(
        f"A curated list of papers related to **{args.topic}** (aggregated from arXiv, Crossref, and Semantic Scholar)."
    )
    lines.append("")
    lines.append(f"> Last updated: **{datetime.now(timezone.utc).strftime('%Y-%m-%d')}**")
    lines.append("")
    lines.append("## Table of Contents")

    category_order = [
        "Security and Trustworthy Agents",
        "System Architecture and Resource Management",
        "Memory, Learning, and Evaluation",
        "Applications and Human Factors",
    ]

    cats = [c for c in category_order if grouped.get(c)]
    for c in cats:
        anchor = re.sub(r"[^a-z0-9]+", "-", c.lower()).strip("-")
        lines.append(f"- [{c}](#{anchor})")

    lines.append("")

    for c in cats:
        lines.append("---")
        lines.append("")
        lines.append(f"## {c}")
        lines.append("")

        items = sorted(grouped[c], key=lambda x: x.get("date", ""), reverse=True)
        for p in items:
            title = p.get("title", "Untitled").strip()
            primary = p.get("url") or p.get("arxiv") or "#"
            date = p.get("date") or "Unknown"
            arxiv = p.get("arxiv") or ""
            github = (p.get("github") or "").strip()

            lines.append(f"+ [{title}]({primary})")

            if has_published_venue(p):
                lines.append(f"  {venue_tag(p)}")

            if arxiv:
                lines.append(f"  {badge('arXiv', arxiv, 'b31b1b')}")

            if github:
                lines.append("  " + badge("GitHub", github, "9cf"))

            lines.append(f"  ({date})")
            lines.append("")

    with open(args.output, "w", encoding="utf-8") as f:
        f.write("\n".join(lines) + "\n")

    print(f"Wrote README -> {args.output}")


if __name__ == "__main__":
    main()


path: scripts/fetch_papers.py
content: #!/usr/bin/env python3
import argparse
import json
import re
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone


def http_get(url: str, timeout: int = 20):
    req = urllib.request.Request(url, headers={"User-Agent": "openclaw-awesome-research-tracker/1.0"})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode("utf-8", errors="ignore")


def fetch_arxiv(topic: str, limit: int):
    q = urllib.parse.quote(topic)
    url = (
        "https://export.arxiv.org/api/query?search_query=all:"
        f"{q}&start=0&max_results={limit}&sortBy=submittedDate&sortOrder=descending"
    )
    txt = http_get(url)
    root = ET.fromstring(txt)
    ns = {"a": "http://www.w3.org/2005/Atom"}
    papers = []
    for e in root.findall("a:entry", ns):
        title = " ".join((e.findtext("a:title", "", ns) or "").split())
        summary = " ".join((e.findtext("a:summary", "", ns) or "").split())
        published = (e.findtext("a:published", "", ns) or "")[:10]
        abs_url = e.findtext("a:id", "", ns)
        pdf_url = ""
        for lk in e.findall("a:link", ns):
            if lk.attrib.get("title") == "pdf":
                pdf_url = lk.attrib.get("href", "")
        papers.append(
            {
                "title": title,
                "abstract": summary,
                "date": published,
                "arxiv": abs_url,
                "pdf": pdf_url,
                "source": "arxiv",
                "venue": "",
                "published_type": "preprint",
                "doi": "",
                "url": abs_url,
            }
        )
    return papers


def fetch_crossref(topic: str, limit: int):
    q = urllib.parse.quote(topic)
    url = f"https://api.crossref.org/works?query={q}&rows={limit}&sort=published&order=desc"
    txt = http_get(url)
    data = json.loads(txt)
    items = data.get("message", {}).get("items", [])
    out = []
    for it in items:
        title_arr = it.get("title") or []
        title = title_arr[0].strip() if title_arr else ""
        if not title:
            continue
        abstract = re.sub(r"<[^>]+>", "", it.get("abstract", "") or "")
        doi = it.get("DOI", "") or ""
        typ = it.get("type", "") or ""
        venue = ""
        if it.get("container-title"):
            venue = it.get("container-title", [""])[0] or ""
        published_type = "published" if typ in {"journal-article", "proceedings-article"} else "unknown"
        y, m, d = "", "", ""
        dp = it.get("issued", {}).get("date-parts", [[]])
        if dp and dp[0]:
            vals = dp[0]
            y = str(vals[0]) if len(vals) >= 1 else ""
            m = str(vals[1]).zfill(2) if len(vals) >= 2 else "01"
            d = str(vals[2]).zfill(2) if len(vals) >= 3 else "01"
        date = f"{y}-{m}-{d}" if y else ""
        link = f"https://doi.org/{doi}" if doi else (it.get("URL", "") or "")
        out.append(
            {
                "title": title,
                "abstract": " ".join(abstract.split()),
                "date": date,
                "arxiv": "",
                "pdf": "",
                "source": "crossref",
                "venue": venue,
                "published_type": published_type,
                "doi": doi,
                "url": link,
            }
        )
    return out


def fetch_semantic(topic: str, limit: int):
    q = urllib.parse.quote(topic)
    fields = "title,abstract,year,venue,url,externalIds,publicationTypes"
    url = f"https://api.semanticscholar.org/graph/v1/paper/search?query={q}&limit={limit}&fields={fields}"
    txt = http_get(url)
    data = json.loads(txt)
    out = []
    for it in data.get("data", []):
        title = (it.get("title") or "").strip()
        if not title:
            continue
        pub_types = it.get("publicationTypes") or []
        ptype = "published" if pub_types else "unknown"
        year = it.get("year")
        date = f"{year}-01-01" if year else ""
        ext = it.get("externalIds") or {}
        arxiv = ""
        if ext.get("ArXiv"):
            arxiv = f"https://arxiv.org/abs/{ext['ArXiv']}"
        out.append(
            {
                "title": title,
                "abstract": " ".join((it.get("abstract") or "").split()),
                "date": date,
                "arxiv": arxiv,
                "pdf": "",
                "source": "semantic_scholar",
                "venue": it.get("venue") or "",
                "published_type": ptype,
                "doi": ext.get("DOI", "") or "",
                "url": it.get("url") or arxiv,
            }
        )
    return out


def dedupe(records):
    seen = set()
    out = []
    for r in records:
        key = (r.get("title", "").strip().lower(), r.get("doi", "").strip().lower())
        if key in seen:
            continue
        seen.add(key)
        out.append(r)
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--topic", required=True)
    ap.add_argument("--max-arxiv", type=int, default=30)
    ap.add_argument("--max-crossref", type=int, default=30)
    ap.add_argument("--max-semantic", type=int, default=30)
    ap.add_argument("--out", required=True)
    args = ap.parse_args()

    rows = []
    errs = []
    for fn, lim in [
        (fetch_arxiv, args.max_arxiv),
        (fetch_crossref, args.max_crossref),
        (fetch_semantic, args.max_semantic),
    ]:
        try:
            rows.extend(fn(args.topic, lim))
        except Exception as e:
            errs.append(f"{fn.__name__}: {e}")

    rows = dedupe(rows)
    rows.sort(key=lambda x: x.get("date", ""), reverse=True)
    payload = {
        "topic": args.topic,
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "errors": errs,
        "count": len(rows),
        "papers": rows,
    }
    with open(args.out, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)
    print(f"Wrote {len(rows)} papers -> {args.out}")
    if errs:
        print("Warnings:")
        for e in errs:
            print("-", e)


if __name__ == "__main__":
    main()


path: scripts/publish_repo.py
content: #!/usr/bin/env python3
import argparse
import os
import shutil
import subprocess
import tempfile
from pathlib import Path


def run(cmd, cwd=None):
    subprocess.run(cmd, cwd=cwd, check=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--owner", required=True)
    ap.add_argument("--name", required=True)
    ap.add_argument("--readme", required=True)
    ap.add_argument("--visibility", choices=["private", "public"], default="private")
    args = ap.parse_args()

    repo = f"{args.owner}/{args.name}"
    vis_flag = "--private" if args.visibility == "private" else "--public"

    with tempfile.TemporaryDirectory(prefix="awesome-research-") as td:
        tdp = Path(td)
        shutil.copyfile(args.readme, tdp / "README.md")
        run(["git", "init", "-b", "main"], cwd=td)
        run(["git", "config", "user.name", "OpenClaw Assistant"], cwd=td)
        run(["git", "config", "user.email", "assistant@local"], cwd=td)
        run(["git", "add", "README.md"], cwd=td)
        run(["git", "commit", "-m", "docs: initialize awesome papers README"], cwd=td)

        exists = subprocess.run(["gh", "repo", "view", repo], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
        if not exists:
            run(["gh", "repo", "create", repo, vis_flag, "--source", td, "--remote", "origin", "--push"], cwd=td)
        else:
            remotes = subprocess.check_output(["git", "remote"], cwd=td).decode()
            if "origin" not in remotes:
                run(["git", "remote", "add", "origin", f"https://github.com/{repo}.git"], cwd=td)
            run(["git", "push", "-u", "origin", "main"], cwd=td)
            run(["gh", "repo", "edit", repo, vis_flag])

    print(f"Published: https://github.com/{repo}")


if __name__ == "__main__":
    main()


Security scan verdicts — ClawScan: suspicious | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (MEDIUM)