Skill: cosformula/douban-sync-skill (version 0.2.2)

--- name: douban-sync description: Export and sync Douban (豆瓣) book/movie/music/game collections to local CSV files (Obsidian-compatible). Use when the user wants to export their Douban reading/watching/listening/gaming history, set up incremental sync via RSS, or manage their Douban data locally. metadata: {"openclaw": {"requires": {"env": ["DOUBAN_USER"]}, "primaryEnv": "DOUBAN_USER"}} --- # Douban Sync Export Douban collections (books, movies, music, games) to CSV and keep them in sync via RSS. ## Two Modes ### 1. Full Export (first time) Use the browser tool to scrape all collection pages. Requires the user to be logged into Douban. ``` browser → douban.com/people/{USER_ID}/{category}?start=0&sort=time&mode=list ``` Categories and URL paths: - Books: `book.douban.com/people/{ID}/collect` (读过), `/do` (在读), `/wish` (想读) - Movies: `movie.douban.com/people/{ID}/collect` (看过), `/do` (在看), `/wish` (想看) - Music: `music.douban.com/people/{ID}/collect` (听过), `/do` (在听), `/wish` (想听) - Games: `www.douban.com/people/{ID}/games?action=collect` (玩过), `=do` (在玩), `=wish` (想玩) Each page shows up to 30 items in list mode (some pages may have fewer due to delisted entries). Paginate with `?start=0,30,60...` — the script uses the paginator's "next" button to determine whether to continue. **Rate limiting:** Wait 2-3 seconds between pages. If blocked, wait 30 seconds and retry. **Scripts:** - `scripts/douban-scraper.mjs` — HTTP-only, no browser needed (may get rate-limited) - `scripts/douban-browser-scraper.mjs` — via Puppeteer CDP, needs a running browser - `scripts/douban-extract.mjs` — generates a browser console script for manual extraction ### 2. Incremental Sync (daily, via RSS) Run `scripts/douban-rss-sync.mjs` — no login needed. ```bash node scripts/douban-rss-sync.mjs ``` **Setup:** Set environment variables: - `DOUBAN_USER` (required): Douban user ID - `DOUBAN_OUTPUT_DIR` (optional): Output root directory, default `~/douban-sync` **Recommended:** Add a daily cron job for automatic sync. ## Output Format Four CSV files per user in the output directory: ``` douban-sync/ └── {user_id}/ ├── 书.csv ├── 影视.csv ├── 音乐.csv └── 游戏.csv ``` CSV columns: ```csv title,url,date,rating,status,comment "书名","https://book.douban.com/subject/12345/","2026-01-15","★★★★★","读过","短评内容" ``` - `status`: 读过/在读/想读, 看过/在看/想看, 听过/在听/想听, 玩过/在玩/想玩 ## Deduplication Both full export and RSS sync deduplicate by Douban URL — safe to run multiple times.

path: scripts/douban-browser-scraper.mjs
content: #!/usr/bin/env node
// Douban full export via CDP browser — outputs CSV
// Connects to an existing browser session (e.g. opened with --remote-debugging-port)

import puppeteer from 'puppeteer-core';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const BROWSER_URL = process.env.BROWSER_URL || 'http://127.0.0.1:18800';
const USER = process.env.DOUBAN_USER;
if (!USER) { console.error('Error: DOUBAN_USER env var is required'); process.exit(1); }
if (!/^[A-Za-z0-9._-]+$/.test(USER)) { console.error('Error: DOUBAN_USER contains invalid characters'); process.exit(1); }
const BASE_DIR = process.env.DOUBAN_OUTPUT_DIR || path.join(os.homedir(), 'douban-sync');
const OUTPUT_DIR = path.join(BASE_DIR, USER);

const CSV_HEADER = 'title,url,date,rating,status,comment\n';

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function csvEscape(str) {
  if (!str) return '';
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return '"' + str.replace(/"/g, '""') + '"';
  }
  return str;
}

function ratingStars(rating) {
  if (!rating || rating === 0) return '';
  return '★'.repeat(rating);
}

const parseScript = `
() => {
  const items = document.querySelectorAll('.list-view .item');
  const results = [];
  for (const item of items) {
    const t = item.querySelector('.title a');
    const title = t ? t.textContent.trim() : '';
    const link = t ? t.getAttribute('href') : '';
    const d = item.querySelector('.date');
    let date = '', rating = 0;
    if (d) {
      const r = d.querySelector('span[class*="rating"]');
      if (r) { const m = r.className.match(/rating(\\d+)-t/); if (m) rating = parseInt(m[1]); }
      const dm = d.textContent.match(/(\\d{4}-\\d{2}-\\d{2})/);
      if (dm) date = dm[1];
    }
    const c = item.querySelector('.comment');
    results.push({ title, link, date, rating, comment: c ? c.textContent.trim() : '' });
  }
  return results;
}
`;

const parseGameScript = `
() => {
  const items = document.querySelectorAll('.common-item');
  const results = [];
  for (const item of items) {
    const t = item.querySelector('.title a');
    const title = t ? t.textContent.trim() : '';
    const link = t ? t.getAttribute('href') : '';
    let date = '', rating = 0;
    const r = item.querySelector('.rating-star');
    if (r) { const m = r.className.match(/allstar(\\d+)/); if (m) rating = parseInt(m[1]) / 10; }
    const d = item.querySelector('.date');
    if (d) { const dm = d.textContent.match(/(\\d{4}-\\d{2}-\\d{2})/); if (dm) date = dm[1]; }
    results.push({ title, link, date, rating, comment: '' });
  }
  return results;
}
`;

const categories = [
  { base: 'https://book.douban.com', path: 'collect', status: '读过', file: '书.csv', type: 'book' },
  { base: 'https://book.douban.com', path: 'do', status: '在读', file: '书.csv', type: 'book' },
  { base: 'https://book.douban.com', path: 'wish', status: '想读', file: '书.csv', type: 'book' },
  { base: 'https://movie.douban.com', path: 'collect', status: '看过', file: '影视.csv', type: 'movie' },
  { base: 'https://movie.douban.com', path: 'do', status: '在看', file: '影视.csv', type: 'movie' },
  { base: 'https://movie.douban.com', path: 'wish', status: '想看', file: '影视.csv', type: 'movie' },
  { base: 'https://music.douban.com', path: 'collect', status: '听过', file: '音乐.csv', type: 'music' },
  { base: 'https://music.douban.com', path: 'do', status: '在听', file: '音乐.csv', type: 'music' },
  { base: 'https://music.douban.com', path: 'wish', status: '想听', file: '音乐.csv', type: 'music' },
  { base: 'https://www.douban.com', path: 'games?action=collect', status: '玩过', file: '游戏.csv', type: 'game' },
  { base: 'https://www.douban.com', path: 'games?action=do', status: '在玩', file: '游戏.csv', type: 'game' },
  { base: 'https://www.douban.com', path: 'games?action=wish', status: '想玩', file: '游戏.csv', type: 'game' },
];

async function scrapeCategory(browser, cat) {
  console.log(`\n=== ${cat.status} (${cat.type}) ===`);
  const page = await browser.newPage();
  const allItems = [];

  try {
    let start = 0;
    while (true) {
      const sep = cat.path.includes('?') ? '&' : '?';
      const url = `${cat.base}/people/${USER}/${cat.path}${sep}start=${start}&sort=time&rating=all&filter=all&mode=list`;
      console.log(`  Fetching start=${start}...`);

      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await sleep(1000);

      const script = cat.type === 'game' ? parseGameScript : parseScript;
      const items = await page.evaluate(new Function('return (' + script + ')()'));
      if (!items || items.length === 0) { console.log('  No items, stopping.'); break; }

      const pageSize = cat.type === 'game' ? 15 : 30;
      console.log(`  Got ${items.length} items`);
      allItems.push(...items);

      const hasNext = await page.evaluate(() => {
        const next = document.querySelector('.paginator .next a');
        return !!next;
      });
      if (!hasNext) break;
      start += pageSize;
      await sleep(2000);
    }
  } catch (err) {
    console.error(`  Error: ${err.message}`);
  } finally {
    await page.close();
  }

  console.log(`  Total: ${allItems.length} items`);
  return allItems;
}

function itemToCsvLine(item, status) {
  return [
    csvEscape(item.title),
    csvEscape(item.link),
    csvEscape(item.date),
    csvEscape(ratingStars(item.rating)),
    csvEscape(status),
    csvEscape(item.comment),
  ].join(',');
}

async function main() {
  console.log('Connecting to browser...');
  const browser = await puppeteer.connect({ browserURL: BROWSER_URL });
  console.log('Connected!');

  try {
    fs.mkdirSync(OUTPUT_DIR, { recursive: true });

    const fileData = {};

    for (const cat of categories) {
      const items = await scrapeCategory(browser, cat);
      if (!fileData[cat.file]) fileData[cat.file] = [];
      for (const item of items) {
        fileData[cat.file].push(itemToCsvLine(item, cat.status));
      }
      await sleep(3000);
    }

    for (const [file, lines] of Object.entries(fileData)) {
      const filePath = path.join(OUTPUT_DIR, file);
      fs.writeFileSync(filePath, CSV_HEADER + lines.join('\n') + '\n', 'utf8');
      console.log(`Written ${lines.length} rows to ${filePath}`);
    }

    console.log('\n✅ All done!');
  } finally {
    browser.disconnect();
  }
}

main().catch(err => {
  console.error('Fatal:', err);
  process.exit(1);
});


path: scripts/douban-extract.mjs
content: #!/usr/bin/env node
// Extract Douban data from browser evaluate results → CSV
// Generates the extraction function to run in browser console

const extractionScript = `
async function extractAllPages(baseUrl, totalExpected) {
  const perPage = 30;
  const results = [];
  const maxPages = Math.ceil(totalExpected / perPage) + 1;

  for (let page = 0; page < maxPages; page++) {
    const start = page * perPage;
    const url = baseUrl + '?start=' + start + '&sort=time&rating=all&filter=all&mode=list';

    try {
      const resp = await fetch(url);
      const html = await resp.text();
      const parser = new DOMParser();
      const doc = parser.parseFromString(html, 'text/html');
      const items = doc.querySelectorAll('.list-view .item');

      if (items.length === 0) break;

      for (const item of items) {
        const t = item.querySelector('.title a');
        const title = t ? t.textContent.trim() : '';
        const link = t ? t.getAttribute('href') : '';
        const d = item.querySelector('.date');
        let date = '', rating = 0;
        if (d) {
          const r = d.querySelector('span[class*="rating"]');
          if (r) { const m = r.className.match(/rating(\\d+)-t/); if (m) rating = parseInt(m[1]); }
          const dm = d.textContent.match(/(\\d{4}-\\d{2}-\\d{2})/);
          if (dm) date = dm[1];
        }
        const c = item.querySelector('.comment');
        results.push({ title, link, date, rating, comment: c ? c.textContent.trim() : '' });
      }

      if (page < maxPages - 1) await new Promise(r => setTimeout(r, 1500));
    } catch (e) {
      console.error('Error fetching page', start, e);
      break;
    }
  }

  return results;
}

// Usage: paste in browser console on douban.com while logged in
// const books = await extractAllPages('https://book.douban.com/people/USER/collect', 400);
// copy(JSON.stringify(books)); // copies to clipboard
`;

console.log(extractionScript);


path: scripts/douban-rss-sync.mjs
content: #!/usr/bin/env node
// Douban RSS → CSV incremental sync
// Pulls RSS feed, parses new entries, appends to corresponding CSV files

import https from 'node:https';
import http from 'node:http';
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const DOUBAN_USER = process.env.DOUBAN_USER;
if (!DOUBAN_USER) { console.error('Error: DOUBAN_USER env var is required'); process.exit(1); }
if (!/^[A-Za-z0-9._-]+$/.test(DOUBAN_USER)) { console.error('Error: DOUBAN_USER contains invalid characters'); process.exit(1); }
const BASE_DIR = process.env.DOUBAN_OUTPUT_DIR || path.join(os.homedir(), 'douban-sync');
const DOUBAN_OUTPUT_DIR = path.join(BASE_DIR, DOUBAN_USER);
const STATE_FILE = process.env.STATE_FILE || path.join(DOUBAN_OUTPUT_DIR, '.douban-rss-state.json');
const RSS_URL = `https://www.douban.com/feed/people/${DOUBAN_USER}/interests`;

// Map RSS title patterns to files + status
const CATEGORY_MAP = [
  { pattern: /^读过/, file: '书.csv', status: '读过', type: 'book' },
  { pattern: /^(?:在读|最近在读)/, file: '书.csv', status: '在读', type: 'book' },
  { pattern: /^想读/, file: '书.csv', status: '想读', type: 'book' },
  { pattern: /^看过/, file: '影视.csv', status: '看过', type: 'movie' },
  { pattern: /^(?:在看|最近在看)/, file: '影视.csv', status: '在看', type: 'movie' },
  { pattern: /^想看/, file: '影视.csv', status: '想看', type: 'movie' },
  { pattern: /^听过/, file: '音乐.csv', status: '听过', type: 'music' },
  { pattern: /^(?:在听|最近在听)/, file: '音乐.csv', status: '在听', type: 'music' },
  { pattern: /^想听/, file: '音乐.csv', status: '想听', type: 'music' },
  { pattern: /^玩过/, file: '游戏.csv', status: '玩过', type: 'game' },
  { pattern: /^(?:在玩|最近在玩)/, file: '游戏.csv', status: '在玩', type: 'game' },
  { pattern: /^想玩/, file: '游戏.csv', status: '想玩', type: 'game' },
];

const CSV_HEADER = 'title,url,date,rating,status,comment\n';

const RATING_MAP = {
  '力荐': '★★★★★',
  '推荐': '★★★★',
  '还行': '★★★',
  '较差': '★★',
  '很差': '★',
};

function fetch(url, redirects = 0) {
  if (redirects > 5) return Promise.reject(new Error('Too many redirects'));
  return new Promise((resolve, reject) => {
    const mod = url.startsWith('https') ? https : http;
    const req = mod.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 15000 }, res => {
      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        const target = new URL(res.headers.location, url).href;
        return fetch(target, redirects + 1).then(resolve, reject);
      }
      if (res.statusCode >= 400) {
        return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
      }
      let data = '';
      res.on('data', c => data += c);
      res.on('end', () => resolve(data));
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
  });
}

function csvEscape(str) {
  if (!str) return '';
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return '"' + str.replace(/"/g, '""') + '"';
  }
  return str;
}

function parseItems(xml) {
  const items = [];
  const itemRegex = /<item>([\s\S]*?)<\/item>/g;
  let match;
  while ((match = itemRegex.exec(xml)) !== null) {
    const block = match[1];
    const get = tag => {
      const m = block.match(new RegExp(`<${tag}[^>]*>(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?<\\/${tag}>`));
      return m ? m[1].trim() : '';
    };
    const title = get('title');
    const link = get('link');
    const guid = get('guid');
    const pubDate = get('pubDate');
    const desc = get('description');

    const ratingMatch = desc.match(/推荐:\s*(力荐|推荐|还行|较差|很差)/);
    const rating = ratingMatch ? RATING_MAP[ratingMatch[1]] || '' : '';

    const commentMatch = desc.match(/短评:\s*([^<]+)/);
    const comment = commentMatch ? commentMatch[1].trim() : '';

    items.push({ title, link, guid, pubDate, rating, comment });
  }
  return items;
}

function loadState() {
  try {
    return JSON.parse(fs.readFileSync(STATE_FILE, 'utf8'));
  } catch {
    return { lastSyncGuids: [] };
  }
}

function saveState(state) {
  fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
}

function extractName(title) {
  for (const { pattern } of CATEGORY_MAP) {
    if (pattern.test(title)) {
      return title.replace(pattern, '');
    }
  }
  return title;
}

function isAlreadyInFile(filePath, link) {
  try {
    const content = fs.readFileSync(filePath, 'utf8');
    const lines = content.split('\n');
    for (let i = 1; i < lines.length; i++) { // skip header
      const cols = lines[i].split(',');
      // URL is second column; strip quotes
      const url = cols[1] ? cols[1].replace(/^"|"$/g, '') : '';
      if (url === link) return true;
    }
    return false;
  } catch {
    return false;
  }
}

function formatDate(pubDateStr) {
  try {
    // Try to extract YYYY-MM-DD directly from source first
    const direct = pubDateStr.match(/(\d{4}-\d{2}-\d{2})/);
    if (direct) return direct[1];
    // Convert to CST (UTC+8) to avoid date shift
    const d = new Date(pubDateStr);
    const cst = new Date(d.getTime() + 8 * 3600000);
    return cst.toISOString().split('T')[0];
  } catch {
    return '';
  }
}

function ensureCsvFile(filePath) {
  if (!fs.existsSync(filePath)) {
    fs.mkdirSync(path.dirname(filePath), { recursive: true });
    fs.writeFileSync(filePath, CSV_HEADER);
  }
}

function appendToCsv(filePath, entry, status) {
  ensureCsvFile(filePath);
  const name = extractName(entry.title);
  const date = formatDate(entry.pubDate);
  const line = [
    csvEscape(name),
    csvEscape(entry.link),
    csvEscape(date),
    csvEscape(entry.rating),
    csvEscape(status),
    csvEscape(entry.comment),
  ].join(',') + '\n';
  fs.appendFileSync(filePath, line);
}

async function main() {
  console.log('Fetching Douban RSS...');
  const xml = await fetch(RSS_URL);
  const items = parseItems(xml);
  console.log(`Found ${items.length} items in RSS feed`);

  const state = loadState();
  const knownGuids = new Set(state.lastSyncGuids || []);
  let newCount = 0;

  for (const item of items) {
    if (knownGuids.has(item.guid)) continue;

    const cat = CATEGORY_MAP.find(c => c.pattern.test(item.title));
    if (!cat) {
      console.log(`  Skipping unknown category: ${item.title}`);
      continue;
    }

    const filePath = path.join(DOUBAN_OUTPUT_DIR, cat.file);

    if (isAlreadyInFile(filePath, item.link)) {
      console.log(`  Already exists: ${item.title}`);
      continue;
    }

    console.log(`  Adding: ${item.title} → ${cat.file}`);
    appendToCsv(filePath, item, cat.status);
    newCount++;
  }

  state.lastSyncGuids = items.map(i => i.guid);
  state.lastSync = new Date().toISOString();
  saveState(state);

  console.log(`Done. ${newCount} new entries added.`);
}

main().catch(err => {
  console.error('Error:', err.message);
  process.exit(1);
});


path: scripts/douban-scraper.mjs
content: #!/usr/bin/env node
// Douban full export scraper — outputs CSV
// Fetches all collection pages via HTTP (no browser needed, but may get rate-limited)

import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const BASE_BOOK = 'https://book.douban.com';
const BASE_MOVIE = 'https://movie.douban.com';
const BASE_MUSIC = 'https://music.douban.com';
const BASE_GAME = 'https://www.douban.com';
const USER = process.env.DOUBAN_USER;
if (!USER) { console.error('Error: DOUBAN_USER env var is required'); process.exit(1); }
if (!/^[A-Za-z0-9._-]+$/.test(USER)) { console.error('Error: DOUBAN_USER contains invalid characters'); process.exit(1); }
const BASE_DIR = process.env.DOUBAN_OUTPUT_DIR || path.join(os.homedir(), 'douban-sync');
const OUTPUT_DIR = path.join(BASE_DIR, USER);

const CSV_HEADER = 'title,url,date,rating,status,comment\n';

const categories = [
  { base: BASE_BOOK, type: 'book', path: 'collect', status: '读过', file: '书.csv' },
  { base: BASE_BOOK, type: 'book', path: 'do', status: '在读', file: '书.csv' },
  { base: BASE_BOOK, type: 'book', path: 'wish', status: '想读', file: '书.csv' },
  { base: BASE_MOVIE, type: 'movie', path: 'collect', status: '看过', file: '影视.csv' },
  { base: BASE_MOVIE, type: 'movie', path: 'do', status: '在看', file: '影视.csv' },
  { base: BASE_MOVIE, type: 'movie', path: 'wish', status: '想看', file: '影视.csv' },
  { base: BASE_MUSIC, type: 'music', path: 'collect', status: '听过', file: '音乐.csv' },
  { base: BASE_MUSIC, type: 'music', path: 'do', status: '在听', file: '音乐.csv' },
  { base: BASE_MUSIC, type: 'music', path: 'wish', status: '想听', file: '音乐.csv' },
  { base: BASE_GAME, type: 'game', path: 'games?action=collect', status: '玩过', file: '游戏.csv' },
  { base: BASE_GAME, type: 'game', path: 'games?action=do', status: '在玩', file: '游戏.csv' },
  { base: BASE_GAME, type: 'game', path: 'games?action=wish', status: '想玩', file: '游戏.csv' },
];

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function csvEscape(str) {
  if (!str) return '';
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return '"' + str.replace(/"/g, '""') + '"';
  }
  return str;
}

function ratingStars(rating) {
  if (!rating || rating === 0) return '';
  return '★'.repeat(rating);
}

function parseListPage(html) {
  const items = [];
  const itemBlocks = html.split(/<div class="item">/);

  for (let i = 1; i < itemBlocks.length; i++) {
    const block = itemBlocks[i];

    // Title and link
    const titleMatch = block.match(/<a[^>]*href="(https:\/\/(?:book|movie|music)\.douban\.com\/subject\/\d+\/)"[^>]*>([\s\S]*?)<\/a>/);
    if (!titleMatch) continue;
    const link = titleMatch[1];
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();

    // Date and rating
    const dateMatch = block.match(/<span class="date">([\s\S]*?)<\/span>/);
    let date = '', rating = 0;
    if (dateMatch) {
      const dm = dateMatch[1].match(/(\d{4}-\d{2}-\d{2})/);
      if (dm) date = dm[1];
      const rm = dateMatch[1].match(/rating(\d+)-t/);
      if (rm) rating = parseInt(rm[1]);
    }

    // Comment
    const commentMatch = block.match(/<span class="comment">([\s\S]*?)<\/span>/);
    const comment = commentMatch ? commentMatch[1].replace(/<[^>]+>/g, '').trim() : '';

    items.push({ title, link, date, rating, comment });
  }
  return items;
}

function parseGamePage(html) {
  const items = [];
  const itemBlocks = html.split(/<div class="common-item">/);

  for (let i = 1; i < itemBlocks.length; i++) {
    const block = itemBlocks[i];

    // Title and link
    const titleMatch = block.match(/<div class="title">\s*<a href="(https:\/\/www\.douban\.com\/game\/\d+\/)">([\s\S]*?)<\/a>/);
    if (!titleMatch) continue;
    const link = titleMatch[1];
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();

    // Rating: allstarNN where NN/10 = stars
    let rating = 0;
    const ratingMatch = block.match(/allstar(\d+)/);
    if (ratingMatch) rating = parseInt(ratingMatch[1]) / 10;

    // Date
    let date = '';
    const dateMatch = block.match(/<span class="date">([\s\S]*?)<\/span>/);
    if (dateMatch) {
      const dm = dateMatch[1].match(/(\d{4}-\d{2}-\d{2})/);
      if (dm) date = dm[1];
    }

    items.push({ title, link, date, rating, comment: '' });
  }
  return items;
}

async function fetchPage(url) {
  const resp = await fetch(url, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
      'Accept': 'text/html',
    }
  });
  if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
  return await resp.text();
}

async function fetchAllItems(base, userPath, type) {
  const allItems = [];
  let start = 0;
  let retries = 0;
  const MAX_RETRIES = 5;
  const hasQuery = userPath.includes('?');
  const isGame = type === 'game';
  const pageSize = isGame ? 15 : 30;

  while (true) {
    const sep = hasQuery ? '&' : '?';
    const url = `${base}/people/${USER}/${userPath}${sep}start=${start}&sort=time&rating=all&filter=all&mode=list`;
    console.log(`Fetching: ${url}`);

    try {
      const html = await fetchPage(url);
      const items = isGame ? parseGamePage(html) : parseListPage(html);

      if (items.length === 0) { console.log('  No items, stopping.'); break; }
      console.log(`  Found ${items.length} items`);
      allItems.push(...items);
      retries = 0;

      if (items.length < pageSize) { console.log(`  Last page.`); break; }
      start += pageSize;
      await sleep(2000);
    } catch (err) {
      console.error(`  Error: ${err.message}`);
      if ((err.message.includes('403') || err.message.includes('418')) && retries < MAX_RETRIES) {
        retries++;
        const delay = 10000 * Math.pow(2, retries - 1);
        console.log(`  Rate limited, retry ${retries}/${MAX_RETRIES}, waiting ${delay/1000}s...`);
        await sleep(delay);
        continue;
      }
      console.error(`  Giving up after ${retries} retries.`);
      break;
    }
  }
  return allItems;
}

function itemToCsvLine(item, status) {
  return [
    csvEscape(item.title),
    csvEscape(item.link),
    csvEscape(item.date),
    csvEscape(ratingStars(item.rating)),
    csvEscape(status),
    csvEscape(item.comment),
  ].join(',');
}

async function main() {
  fs.mkdirSync(OUTPUT_DIR, { recursive: true });

  // Group by output file to avoid overwriting
  const fileData = {};

  for (const cat of categories) {
    console.log(`\n=== ${cat.status} (${cat.type}) ===`);
    const items = await fetchAllItems(cat.base, cat.path, cat.type);
    console.log(`Total: ${items.length} items`);

    if (!fileData[cat.file]) fileData[cat.file] = [];
    for (const item of items) {
      fileData[cat.file].push(itemToCsvLine(item, cat.status));
    }

    await sleep(3000);
  }

  for (const [file, lines] of Object.entries(fileData)) {
    const filePath = path.join(OUTPUT_DIR, file);
    fs.writeFileSync(filePath, CSV_HEADER + lines.join('\n') + '\n', 'utf8');
    console.log(`Written ${lines.length} rows to ${filePath}`);
  }

  console.log('\nDone!');
}

main().catch(console.error);


path: scripts/lib.mjs
content: // Shared utility functions extracted for testing.

export const RATING_MAP = {
  '力荐': '★★★★★',
  '推荐': '★★★★',
  '还行': '★★★',
  '较差': '★★',
  '很差': '★',
};

export const CATEGORY_MAP = [
  { pattern: /^读过/, file: '书.csv', status: '读过', type: 'book' },
  { pattern: /^(?:在读|最近在读)/, file: '书.csv', status: '在读', type: 'book' },
  { pattern: /^想读/, file: '书.csv', status: '想读', type: 'book' },
  { pattern: /^看过/, file: '影视.csv', status: '看过', type: 'movie' },
  { pattern: /^(?:在看|最近在看)/, file: '影视.csv', status: '在看', type: 'movie' },
  { pattern: /^想看/, file: '影视.csv', status: '想看', type: 'movie' },
  { pattern: /^听过/, file: '音乐.csv', status: '听过', type: 'music' },
  { pattern: /^(?:在听|最近在听)/, file: '音乐.csv', status: '在听', type: 'music' },
  { pattern: /^想听/, file: '音乐.csv', status: '想听', type: 'music' },
  { pattern: /^玩过/, file: '游戏.csv', status: '玩过', type: 'game' },
  { pattern: /^(?:在玩|最近在玩)/, file: '游戏.csv', status: '在玩', type: 'game' },
  { pattern: /^想玩/, file: '游戏.csv', status: '想玩', type: 'game' },
];

export function csvEscape(str) {
  if (!str) return '';
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return '"' + str.replace(/"/g, '""') + '"';
  }
  return str;
}

export function parseItems(xml) {
  const items = [];
  const itemRegex = /<item>([\s\S]*?)<\/item>/g;
  let match;
  while ((match = itemRegex.exec(xml)) !== null) {
    const block = match[1];
    const get = tag => {
      const m = block.match(new RegExp(`<${tag}[^>]*>(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?<\\/${tag}>`));
      return m ? m[1].trim() : '';
    };
    const title = get('title');
    const link = get('link');
    const guid = get('guid');
    const pubDate = get('pubDate');
    const desc = get('description');

    const ratingMatch = desc.match(/推荐:\s*(力荐|推荐|还行|较差|很差)/);
    const rating = ratingMatch ? RATING_MAP[ratingMatch[1]] || '' : '';

    const commentMatch = desc.match(/短评:\s*([^<]+)/);
    const comment = commentMatch ? commentMatch[1].trim() : '';

    items.push({ title, link, guid, pubDate, rating, comment });
  }
  return items;
}

export function matchCategory(title) {
  for (const cat of CATEGORY_MAP) {
    if (cat.pattern.test(title)) return cat;
  }
  return null;
}


path: scripts/migrate-md-to-csv.mjs
content: #!/usr/bin/env node
// Migrate existing Douban Markdown files to CSV format
// Handles both formats:
//   1. Full export: ### Title\n- 作者：...\n- 我的评分：...\n- 标记日期：...\n- 短评：...\n- 链接：...
//   2. RSS incremental: - [Title](url) | date | rating | "comment"

import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const DOUBAN_USER = process.env.DOUBAN_USER;
if (!DOUBAN_USER) { console.error('Error: DOUBAN_USER env var is required'); process.exit(1); }
if (!/^[A-Za-z0-9._-]+$/.test(DOUBAN_USER)) { console.error('Error: DOUBAN_USER contains invalid characters'); process.exit(1); }
const BASE_DIR = process.env.DOUBAN_OUTPUT_DIR || path.join(os.homedir(), 'douban-sync');
const DOUBAN_DIR = BASE_DIR; // read md from base dir
const OUTPUT_DIR = path.join(BASE_DIR, DOUBAN_USER); // write csv to user subdir
const CSV_HEADER = 'title,url,date,rating,status,comment\n';

const FILE_MAP = [
  { md: '读过的书.md', csv: '书.csv', status: '读过' },
  { md: '在读的书.md', csv: '书.csv', status: '在读' },
  { md: '想读的书.md', csv: '书.csv', status: '想读' },
  { md: '看过的影视.md', csv: '影视.csv', status: '看过' },
  { md: '在看的影视.md', csv: '影视.csv', status: '在看' },
  { md: '想看的影视.md', csv: '影视.csv', status: '想看' },
  { md: '听过的音乐.md', csv: '音乐.csv', status: '听过' },
  { md: '在听的音乐.md', csv: '音乐.csv', status: '在听' },
  { md: '想听的音乐.md', csv: '音乐.csv', status: '想听' },
];

function csvEscape(str) {
  if (!str) return '';
  if (str.includes(',') || str.includes('"') || str.includes('\n')) {
    return '"' + str.replace(/"/g, '""') + '"';
  }
  return str;
}

function emojiToStars(emoji) {
  if (!emoji) return '';
  const count = (emoji.match(/⭐/g) || []).length;
  if (count > 0) return '★'.repeat(count);
  // Already ★ format
  const starCount = (emoji.match(/★/g) || []).length;
  if (starCount > 0) return emoji;
  return '';
}

function parseMdFile(filePath, status) {
  if (!fs.existsSync(filePath)) return [];

  const content = fs.readFileSync(filePath, 'utf8');
  const lines = content.split('\n');
  const items = [];

  let currentItem = null;

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];

    // Format 2: RSS single-line - [Title](url) | date | rating | "comment"
    const rssMatch = line.match(/^- \[(.+?)\]\((https?:\/\/[^\)]+)\)\s*(?:\|\s*(\S+))?\s*(?:\|\s*(★+))?\s*(?:\|\s*"(.+?)")?$/);
    if (rssMatch) {
      items.push({
        title: rssMatch[1],
        url: rssMatch[2],
        date: rssMatch[3] || '',
        rating: rssMatch[4] || '',
        status,
        comment: rssMatch[5] || '',
      });
      continue;
    }

    // Format 1: Full export ### Title
    if (line.startsWith('### ')) {
      // Save previous item
      if (currentItem && currentItem.url) {
        items.push(currentItem);
      }
      currentItem = { title: line.slice(4).trim(), url: '', date: '', rating: '', status, comment: '' };
      continue;
    }

    if (currentItem) {
      if (line.startsWith('- 链接：')) {
        currentItem.url = line.slice(5).trim();
      } else if (line.startsWith('- 标记日期：')) {
        currentItem.date = line.slice(7).trim();
      } else if (line.startsWith('- 我的评分：')) {
        const raw = line.slice(7).trim();
        if (raw !== '未评分') currentItem.rating = emojiToStars(raw);
      } else if (line.startsWith('- 短评：')) {
        currentItem.comment = line.slice(5).trim();
      }
      // Also handle inline link format in full export: - [Title](url) | date | rating
      const inlineMatch = line.match(/^- \[(.+?)\]\((https?:\/\/[^\)]+)\)/);
      if (inlineMatch && !currentItem.url) {
        currentItem.url = inlineMatch[2];
      }
    }
  }

  // Don't forget last item
  if (currentItem && currentItem.url) {
    items.push(currentItem);
  }

  return items;
}

function main() {
  const csvData = {}; // file -> lines[]

  for (const mapping of FILE_MAP) {
    const mdPath = path.join(DOUBAN_DIR, mapping.md);
    if (!fs.existsSync(mdPath)) {
      console.log(`Skipping ${mapping.md} (not found)`);
      continue;
    }

    const items = parseMdFile(mdPath, mapping.status);
    console.log(`${mapping.md}: ${items.length} items → ${mapping.csv}`);

    if (!csvData[mapping.csv]) csvData[mapping.csv] = [];

    for (const item of items) {
      csvData[mapping.csv].push(item);
    }
  }

  // Deduplicate by URL within each file
  for (const [file, items] of Object.entries(csvData)) {
    const seen = new Set();
    const deduped = [];
    for (const item of items) {
      if (seen.has(item.url)) continue;
      seen.add(item.url);
      deduped.push([
        csvEscape(item.title),
        csvEscape(item.url),
        csvEscape(item.date),
        csvEscape(item.rating),
        csvEscape(item.status),
        csvEscape(item.comment),
      ].join(','));
    }

    fs.mkdirSync(OUTPUT_DIR, { recursive: true });
    const filePath = path.join(OUTPUT_DIR, file);
    fs.writeFileSync(filePath, CSV_HEADER + deduped.join('\n') + '\n', 'utf8');
    console.log(`Written ${deduped.length} rows to ${filePath} (deduped from ${items.length})`);
  }

  console.log('\nMigration complete!');
  console.log('You can now delete the old .md files if the CSV data looks correct.');
}

main();


Security scan verdicts — ClawScan: clean | static analysis: suspicious | VirusTotal: clean | SkillSpector: clean (LOW)