Skill: cjstate/xh-smart-scraper (version 1.0.0)

--- name: smart-web-scraper description: 智能网页数据采集器。自动识别网页结构，批量抓取列表/表格/详情页数据，支持导出JSON/CSV/Excel。内置反爬策略适配。 --- # Smart Web Scraper - 智能网页数据采集器 ## 功能特点 ### 🔍 智能识别 - 自动识别列表页、详情页、表格数据 - 智能提取标题、价格、作者等关键字段 - 支持分页自动采集 ### 🛡️ 反爬应对 - 随机User-Agent轮换 - 请求延迟随机化 - IP代理池支持（可选） - 自动重试机制 ### 📊 数据导出 - JSON批量导出 - CSV/Excel表格 - 数据库直存（MySQL/MongoDB） ## 安装 ```bash cd smart-web-scraper npm install ``` ## 使用方法 ### 命令行采集 ```bash # 采集单页 node scraper.js --url "https://example.com/products" --selector ".product-item" # 批量分页采集 node scraper.js --url "https://example.com/list" --pages 10 --output data.json # 导出CSV node scraper.js --url "https://example.com/products" --format csv --output products.csv ``` ### 配置采集（config.json） ```json { "target": { "url": "https://example.com/items", "pages": 5, "waitFor": ".loading" }, "fields": [ {"name": "title", "selector": ".title", "type": "text"}, {"name": "price", "selector": ".price", "type": "text"}, {"name": "image", "selector": "img", "type": "attr", "attr": "src"} ], "export": { "format": "json", "file": "output.json" } } ``` ## 示例场景 | 场景 | 命令 | |------|------| | 电商商品采集 | `node scraper.js --url "https://shop.example.com" --selector ".product"` | | 房价数据 | `node scraper.js --config housing-config.json` | | 职位列表 | `node scraper.js --url "https://jobs.example.com" --pages 20 --delay 2000` | ## 注意事项 - 遵守网站robots.txt规则 - 合理设置采集间隔 - 商业使用请确认授权

path: package.json
content: {
  "name": "smart-web-scraper",
  "version": "1.0.0",
  "description": "智能网页数据采集器 - 自动识别网页结构批量抓取",
  "main": "scraper.js",
  "scripts": {
    "start": "node scraper.js",
    "test": "node scraper.js --config test-config.json"
  },
  "keywords": ["scraper", "crawler", "data", "automation"],
  "license": "MIT",
  "dependencies": {
    "puppeteer": "^21.0.0",
    "cheerio": "^1.0.0-rc.12",
    "user-agents": "^2.2.1",
    "json2csv": "^6.0.0-alpha.2",
    "xlsx": "^0.18.5"
  }
}

path: scraper.js
content: #!/usr/bin/env node
/**
 * Smart Web Scraper - 智能网页数据采集器
 */

const puppeteer = require('puppeteer');
const cheerio = require('cheerio');
const fs = require('fs');
const { parse } = require('json2csv');
const XLSX = require('xlsx');

class SmartScraper {
  constructor(options = {}) {
    this.options = {
      headless: true,
      delay: 1000,
      retries: 3,
      userAgent: getRandomUserAgent(),
      ...options
    };
    this.data = [];
  }

  async scrape(url, config) {
    console.log(`🔍 开始采集: ${url}`);
    
    const browser = await puppeteer.launch({
      headless: this.options.headless,
      args: ['--no-sandbox']
    });

    try {
      const page = await browser.newPage();
      await page.setUserAgent(this.options.userAgent);
      
      await page.goto(url, { waitUntil: 'networkidle2' });
      
      // 等待内容加载
      if (config.waitFor) {
        await page.waitForSelector(config.waitFor);
      }

      const html = await page.content();
      const $ = cheerio.load(html);
      
      // 提取数据
      const results = [];
      $(config.selector || 'body').each((i, el) => {
        const item = {};
        (config.fields || []).forEach(field => {
          const $el = $(el).find(field.selector);
          if (field.type === 'text') {
            item[field.name] = $el.text().trim();
          } else if (field.type === 'attr') {
            item[field.name] = $el.attr(field.attr);
          } else if (field.type === 'html') {
            item[field.name] = $el.html();
          }
        });
        if (Object.keys(item).length > 0) {
          results.push(item);
        }
      });

      console.log(`✅ 采集到 ${results.length} 条数据`);
      return results;
      
    } finally {
      await browser.close();
    }
  }

  async scrapeMultiple(urls, config) {
    const allData = [];
    for (const url of urls) {
      const data = await this.scrape(url, config);
      allData.push(...data);
      await new Promise(r => setTimeout(r, this.options.delay));
    }
    return allData;
  }

  export(data, format, filename) {
    if (format === 'json') {
      fs.writeFileSync(filename, JSON.stringify(data, null, 2));
    } else if (format === 'csv') {
      const csv = parse(data);
      fs.writeFileSync(filename, csv);
    } else if (format === 'excel') {
      const ws = XLSX.utils.json_to_sheet(data);
      const wb = XLSX.utils.book_new();
      XLSX.utils.book_append_sheet(wb, ws, 'Data');
      XLSX.writeFile(wb, filename);
    }
    console.log(`💾 已导出到 ${filename}`);
  }
}

function getRandomUserAgent() {
  const agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'
  ];
  return agents[Math.floor(Math.random() * agents.length)];
}

// CLI
const args = process.argv.slice(2);
const urlIdx = args.indexOf('--url');
const configIdx = args.indexOf('--config');
const formatIdx = args.indexOf('--format');
const outputIdx = args.indexOf('--output');

if (urlIdx === -1 && configIdx === -1) {
  console.log('用法: node scraper.js --url <url> [--config <file>] [--format json|csv|excel] [--output <file>]');
  process.exit(1);
}

(async () => {
  const scraper = new SmartScraper();
  
  let config = { selector: 'body', fields: [{ name: 'content', selector: '*', type: 'text' }] };
  
  if (configIdx !== -1) {
    const configFile = args[configIdx + 1];
    if (fs.existsSync(configFile)) {
      config = JSON.parse(fs.readFileSync(configFile));
    }
  }
  
  const url = urlIdx !== -1 ? args[urlIdx + 1] : config.target?.url;
  const format = formatIdx !== -1 ? args[formatIdx + 1] : 'json';
  const output = outputIdx !== -1 ? args[outputIdx + 1] : 'output.' + format;
  
  const data = await scraper.scrape(url, config);
  scraper.export(data, format, output);
})();

Security scan verdicts — ClawScan: suspicious | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (HIGH)