Skill: 10oss/skrape (version 1.1.1)

--- name: Skrape description: Ethical web data extraction with robots exclusion protocol adherence, throttled scraping requests, and privacy-compliant handling ("Scrape responsibly!"). --- ## Respect Creative Work - **Design & text copying**: Avoid copying design elements or substantial portions of text; while facts and data aren't typically protected by copyright, their presentation (website layouts, specific text, compilations) often is. - **Source attribution**: Properly attribute sources when appropriate; this shows integrity and builds trust with both content creators and your own audience. - **Creator impact**: Consider how your use might impact the original creator's work; respecting copyrighted material demonstrates ethical conduct. ## Pre-Extraction Verification Steps **I. Access Authorization** — Retrieve `{domain}/robots.txt` and review `/terms` or `/tos` endpoints. Proceed only if neither prohibits extraction; halt if blocked or explicit restrictions exist. **II. Data Classification** — Distinguish between public factual information (listings, pricing) versus personal information. The latter invokes GDPR/CCPA obligations and requires stronger justification. **III. Preferred Channels** — Check whether the platform offers an API. If available, use it instead of direct extraction. Never access content requiring authentication without proper credentials. ## Operational Conduct & Compliance - **Request discipline**: Throttle at 2-3 seconds minimum, honor 429 with progressive backoff, maintain connection pooling, and use authentic User-Agent with contact email. - **Access boundaries**: robots.txt disregard carries uncertain legal standing (Meta v. Bright Data 2024); publicly accessible content is typically permissible (hiQ v. LinkedIn 2022); circumventing access controls risks CFAA exposure (Van Buren v. US 2021). - **Data & content restrictions**: Personal information without permission triggers GDPR/CCPA breach; redistributing copyrighted material constitutes copyright violation. ## Information Stewardship - **PII & profiling restrictions**: Remove personal information promptly and avoid correlating data to identify individuals. - **Limit retention**: Store only necessary data, purge the rest. - **Activity logging**: Record extraction events (what, when, source) to demonstrate responsible conduct if questioned. Implementation patterns and robots.txt evaluation logic in `code.md`

path: code.md
content: # Skrape — Implementation Patterns (JavaScript)

## robots.txt Verification (First Step)

```javascript
const https = require('https');
const http = require('http');
const { URL } = require('url');

function verifyRobotsAccess(url, agentLabel = '*') {
  return new Promise((resolve, reject) => {
    const parsed = new URL(url);
    const robotsUrl = `${parsed.protocol}//${parsed.host}/robots.txt`;
    
    const fetchRobots = (protocol) => {
      return new Promise((resolve, reject) => {
        protocol.get(robotsUrl, (res) => {
          let data = '';
          res.on('data', chunk => data += chunk);
          res.on('end', () => {
            try {
              const allowed = evaluateRobotsRules(data, agentLabel, parsed.pathname);
              resolve(allowed);
            } catch (e) {
              resolve(true); // Missing robots.txt = permitted
            }
          });
        }).on('error', () => resolve(true));
      });
    };
    
    fetchRobots(parsed.protocol === 'https:' ? https : http)
      .then(resolve)
      .catch(() => resolve(true));
  });
}

function evaluateRobotsRules(content, agentLabel, path) {
  const lines = content.split('\n');
  let activeAgent = null;
  let blockedPaths = [];
  let allowedPaths = [];
  
  for (const line of lines) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith('#')) continue;
    
    const [key, value] = trimmed.split(':', 2).map(s => s.trim());
    
    if (key?.toLowerCase() === 'user-agent') {
      if (activeAgent !== null && agentMatches(activeAgent, agentLabel)) {
        for (const p of blockedPaths) {
          if (path.startsWith(p)) return false;
        }
      }
      activeAgent = value;
      blockedPaths = [];
      allowedPaths = [];
    } else if (key?.toLowerCase() === 'disallow') {
      if (value) blockedPaths.push(value);
    } else if (key?.toLowerCase() === 'allow') {
      if (value) allowedPaths.push(value);
    }
  }
  
  // Evaluate final user-agent block
  if (activeAgent !== null && agentMatches(activeAgent, agentLabel)) {
    for (const p of blockedPaths) {
      if (path.startsWith(p)) return false;
    }
  }
  
  return true;
}

function agentMatches(ruleAgent, requestAgent) {
  if (ruleAgent === '*') return true;
  return ruleAgent.toLowerCase() === requestAgent.toLowerCase();
}
```

## Connection Manager

```javascript
const https = require('https');
const http = require('http');
const { URL } = require('url');

function buildConnectionManager(contactEmail) {
  const agentOptions = {
    keepAlive: true,
    maxSockets: 1,
    timeout: 30000,
  };
  
  const httpsAgent = new https.Agent(agentOptions);
  const httpAgent = new http.Agent(agentOptions);
  
  return {
    userAgent: `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 (contact: ${contactEmail})`,
    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    acceptLanguage: 'en-US,en;q=0.9',
    httpsAgent,
    httpAgent,
    
    get(url) {
      return new Promise((resolve, reject) => {
        const parsed = new URL(url);
        const protocol = parsed.protocol === 'https:' ? https : http;
        const agent = parsed.protocol === 'https:' ? this.httpsAgent : this.httpAgent;
        
        const req = protocol.get(url, {
          headers: {
            'User-Agent': this.userAgent,
            'Accept': this.accept,
            'Accept-Language': this.acceptLanguage,
          },
          agent,
        }, (res) => {
          let data = '';
          res.on('data', chunk => data += chunk);
          res.on('end', () => {
            resolve({
              status: res.statusCode,
              headers: res.headers,
              text: data,
            });
          });
        });
        
        req.on('error', reject);
        req.setTimeout(30000, () => {
          req.destroy();
          reject(new Error('Request timeout'));
        });
      });
    },
  };
}
```

## Throttled Request Handler

```javascript
const { Console } = require('console');

const logger = new Console({
  stdout: process.stdout,
  stderr: process.stderr,
});

async function executeThrottledRequest(manager, url, baseDelay = 2.0, attemptLimit = 5) {
  /** Execute with throttling, backoff strategy, and audit logging. */
  
  for (let attempt = 0; attempt < attemptLimit; attempt++) {
    // Courteous pause with randomization
    const pause = baseDelay + Math.random() * 0.5;
    await wait(pause * 1000);
    
    const response = await manager.get(url);
    
    // Audit record
    logger.info(`SCRAPE url=${url} status=${response.status}`);
    
    // Inspect rate limit indicators
    const remaining = response.headers['x-ratelimit-remaining'];
    if (remaining && parseInt(remaining, 10) < 5) {
      logger.warn(`Rate limit depleted: ${remaining} remaining`);
      await wait(10000); // Preventive slowdown
    }
    
    // Handle 429 responses
    if (response.status === 429) {
      const retryAfter = response.headers['retry-after'];
      const pauseDuration = retryAfter && !isNaN(parseInt(retryAfter, 10)) 
        ? parseInt(retryAfter, 10) * 1000 
        : 60000;
      logger.warn(`429 received, pausing ${pauseDuration / 1000}s`);
      await wait(pauseDuration);
      continue;
    }
    
    // Success or client error (skip retry for 4xx except 429)
    if (response.status < 500) {
      return response;
    }
    
    // Server error: progressive backoff
    const pauseDuration = Math.min(Math.pow(2, attempt) * 1000 + Math.random() * 1000, 60000);
    logger.warn(`5xx error, retry in ${(pauseDuration / 1000).toFixed(1)}s`);
    await wait(pauseDuration);
  }
  
  throw new Error(`Failed after ${attemptLimit} attempts: ${url}`);
}

function wait(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}
```

## Complete Workflow

```javascript
const { verifyRobotsAccess, buildConnectionManager, executeThrottledRequest } = require('./scrape');

async function runScraper() {
  const contact = '[REDACTED_SECRET]';
  const target = 'https://example.com/products';
  
  // 1. Verify robots.txt
  const permitted = await verifyRobotsAccess(target);
  if (!permitted) {
    throw new Error('Blocked by robots.txt');
  }
  
  // 2. Initialize connection manager
  const manager = buildConnectionManager(contact);
  
  // 3. Execute throttled request
  const response = await executeThrottledRequest(manager, target);
  
  // 4. Process (implement your logic)
  console.log(`Retrieved ${response.text.length} bytes`);
}

runScraper().catch(console.error);
```


Security scan verdicts — ClawScan: clean | static analysis: clean | VirusTotal: clean | SkillSpector: clean (LOW)