Skill: ffagen/ffagen-minimax-vision-scraper (version 1.0.0)

--- name: ffagen__minimax-vision-scraper description: Playwright截图 + MiniMax图像理解的高级网页抓取skill。绕过反爬虫，直接用AI理解截图内容。 version: 1.0.0 author: ffagen credits: | 技术参考： - playwright-scraper-skill by Simon Chan (https://clawhub.ai/skills/scraper) - 反爬虫技术 - minimax-coding-plan-mcp by MiniMax - 图像理解API --- # MiniMax Vision Scraper Playwright 截图 + MiniMax 图像理解，绕过反爬虫，直接用 AI 提取页面内容。 --- ## 核心优势 | 对比项 | 传统抓取 | Vision Scraper | |--------|----------|----------------| | 反爬虫 | ❌ 易被屏蔽 | ✅ 截图绕过检测 | | JS渲染 | ⚠️ 复杂 | ✅ 截图即完整 | | 内容理解 | ❌ 需解析HTML | ✅ AI直接理解 | | 动态内容 | ❌ 难抓 | ✅ 截图即所见 | --- ## 工作流程 ``` URL → Playwright截图(Chrome) → MiniMax VLM图像理解 → AI分析结果 ``` --- ## 安装 ```bash cd ~/.openclaw/workspace/skills/ffagen__minimax-vision-scraper npm install playwright # 无需安装 chromium，直接使用系统已装的 Google Chrome ``` --- ## 使用方式 ### 直接调用 ```bash node scripts/screenshot.js <URL> [prompt] # 示例 node scripts/screenshot.js "https://news.sina.com.cn" "提取今日头条新闻" node scripts/screenshot.js "https://finance.sina.com.cn" "提取股票行情数据" ``` ### 环境变量 | 变量 | 默认值 | 说明 | |------|--------|------| | WAIT_TIME | 5000 | 等待时间(ms) | | HEADLESS | true | 是否无头模式 | | SCREENSHOT_PATH | /tmp/screenshot-*.png | 截图保存路径 | | MINIMAX_API_KEY | (内置) | MiniMax API Key | | MINIMAX_API_HOST | api.minimaxi.com | API地址 | ### 在OpenClaw中调用 ``` 用 ffagen__minimax-vision-scraper 抓取 https://example.com，分析页面内容 ``` --- ## 反爬虫技术（致敬 playwright-scraper-skill） ✅ 隐藏 `navigator.webdriver` ✅ 真实 iPhone User-Agent ✅ 移动端视口 (375x812) ✅ 隐藏 Chrome 自动化特征 ✅ 模拟 permissions.query ✅ Cloudflare 检测 + 额外等待 --- ## 示例 **抓取财经新闻：** ```bash node scripts/screenshot.js "https://finance.sina.com.cn" "提取所有财经新闻标题和摘要" ``` **抓取股票数据：** ```bash node scripts/screenshot.js "https://stock.finance.sina.com.cn" "提取大盘指数和涨跌数据" ``` **抓取商品信息：** ```bash node scripts/screenshot.js "https://item.jd.com/100000.html" "提取商品名称、价格、评价" ``` --- ## 技术说明 - **截图引擎**：使用系统已装的 Google Chrome（无需额外下载） - **图像理解**：MiniMax VLM API (`/v1/coding_plan/vlm`) - **反爬策略**：Playwright Stealth 模式，隐藏自动化特征

path: package-lock.json
content: {
  "name": "minimax-vision-scraper",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "dependencies": {
        "playwright": "^1.59.1"
      }
    },
    "node_modules/fsevents": {
      "version": "2.3.2",
      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
      "integrity": [REDACTED_SECRET],
      "hasInstallScript": true,
      "license": "MIT",
      "optional": true,
      "os": [
        "darwin"
      ],
      "engines": {
        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
      }
    },
    "node_modules/playwright": {
      "version": "1.59.1",
      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz",
      "integrity": [REDACTED_SECRET],
      "license": "Apache-2.0",
      "dependencies": {
        "playwright-core": "1.59.1"
      },
      "bin": {
        "playwright": "cli.js"
      },
      "engines": {
        "node": ">=18"
      },
      "optionalDependencies": {
        "fsevents": "2.3.2"
      }
    },
    "node_modules/playwright-core": {
      "version": "1.59.1",
      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz",
      "integrity": [REDACTED_SECRET],
      "license": "Apache-2.0",
      "bin": {
        "playwright-core": "cli.js"
      },
      "engines": {
        "node": ">=18"
      }
    }
  }
}


path: package.json
content: {
  "dependencies": {
    "playwright": "^1.59.1"
  }
}


path: scripts/screenshot.js
content: #!/usr/bin/env node
/**
 * MiniMax Vision Scraper
 * Playwright截图 + MiniMax图像理解
 * 
 * Usage: node screenshot.js <URL> [prompt]
 * 
 * 依赖：
 * - playwright (已安装)
 * - MINIMAX_API_KEY (环境变量)
 */

const { chromium } = require('playwright');
const fs = require('fs');
const path = require('path');
const https = require('https');
const http = require('http');

// API配置
const [REDACTED_SECRET] || '[REDACTED_SECRET]';
const API_HOST = process.env.MINIMAX_API_HOST || 'api.minimaxi.com';
const API_PATH = '/v1/coding_plan/vlm';

const url = process.argv[2];
const prompt = process.argv[3] || '分析这张截图的内容，提取关键信息';
const waitTime = parseInt(process.env.WAIT_TIME || '5000');
const headless = process.env.HEADLESS !== 'false';
const screenshotPath = process.env.SCREENSHOT_PATH || `/tmp/screenshot-${Date.now()}.png`;

if (!url) {
    console.error('❌ 请提供 URL');
    console.error('用法: node screenshot.js <URL> [prompt]');
    process.exit(1);
}

const defaultUA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1';

async function takeScreenshot() {
    console.log('🕷️ 启动 MiniMax Vision Scraper...');
    console.log(`🔗 URL: ${url}`);
    
    const startTime = Date.now();
    
    const browser = await chromium.launch({
        headless: headless,
        executablePath: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-blink-features=AutomationControlled',
            '--disable-features=IsolateOrigins,site-per-process',
        ],
    });
    
    const context = await browser.newContext({
        userAgent: defaultUA,
        locale: 'zh-CN',
        viewport: { width: 375, height: 812 },
        extraHTTPHeaders: {
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        },
    });
    
    // 隐藏自动化特征
    await context.addInitScript(() => {
        Object.defineProperty(navigator, 'webdriver', { get: () => false });
        window.chrome = { runtime: {} };
        const originalQuery = window.navigator.permissions.query;
        window.navigator.permissions.query = (parameters) => (
            parameters.name === 'notifications' ?
                Promise.resolve({ state: Notification.permission }) :
                originalQuery(parameters)
        );
    });
    
    const page = await context.newPage();
    
    console.log('📱 导航到目标页面...');
    try {
        const response = await page.goto(url, {
            waitUntil: 'domcontentloaded',
            timeout: 30000,
        });
        console.log(`📡 HTTP Status: ${response.status()}`);
    } catch (error) {
        console.error(`⚠️ 导航异常: ${error.message}`);
    }
    
    // 等待内容加载
    console.log(`⏳ 等待 ${waitTime}ms...`);
    await page.waitForTimeout(waitTime);
    
    // 检测 Cloudflare
    const isCloudflare = await page.evaluate(() => {
        return document.body.innerText.includes('Checking your browser') ||
               document.body.innerText.includes('Just a moment');
    });
    
    if (isCloudflare) {
        console.log('🛡️ 检测到 Cloudflare，等待额外 10s...');
        await page.waitForTimeout(10000);
    }
    
    // 截图
    console.log(`📸 截图保存到: ${screenshotPath}`);
    await page.screenshot({ path: screenshotPath, fullPage: false });
    
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
    console.log(`⏱️ 截图耗时: ${elapsed}s`);
    
    await browser.close();
    
    return screenshotPath;
}

function callMiniMaxVision(imagePath, prompt) {
    return new Promise((resolve, reject) => {
        // 读取图片并转 base64
        const imageBuffer = fs.readFileSync(imagePath);
        const base64Image = imageBuffer.toString('base64');
        const mimeType = 'image/png';
        
        const payload = {
            prompt: prompt,
            image_url: `data:${mimeType};base64,${base64Image}`
        };
        
        const body = JSON.stringify(payload);
        
        const options = {
            hostname: API_HOST,
            port: 443,
            path: API_PATH,
            method: 'POST',
            headers: {
                'Authorization': `Bearer ${API_KEY}`,
                'Content-Type': 'application/json',
                'Content-Length': Buffer.byteLength(body)
            }
        };
        
        console.log('🤖 调用 MiniMax 图像理解...');
        const startTime = Date.now();
        
        const req = https.request(options, (res) => {
            let data = '';
            res.on('data', chunk => data += chunk);
            res.on('end', () => {
                const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
                try {
                    const json = JSON.parse(data);
                    const result = json.choices?.[0]?.message?.content || JSON.stringify(json);
                    console.log(`⏱️ MiniMax 响应耗时: ${elapsed}s`);
                    resolve(result);
                } catch (e) {
                    resolve(data);
                }
            });
        });
        
        req.on('error', reject);
        req.write(body);
        req.end();
    });
}

(async () => {
    try {
        const screenshotPath = await takeScreenshot();
        const result = await callMiniMaxVision(screenshotPath, prompt);
        
        console.log('\n✅ 分析结果:');
        console.log(result);
        
        // 清理临时截图
        fs.unlinkSync(screenshotPath);
        
        process.exit(0);
    } catch (error) {
        console.error('❌ 错误:', error.message);
        process.exit(1);
    }
})();


Security scan verdicts — ClawScan: suspicious | static analysis: clean | VirusTotal: clean | SkillSpector: suspicious (CRITICAL)