fix: implement robust fetching with retries and exponential backoff for benchmarks and providers

scripts/fetch-benchmarks.js

@@ -34,19 +34,12 @@
 const fs   = require('fs');
 const path = require('path');
 const yaml = require('js-yaml');
+const { getJson, getText } = require('./fetch-utils');
 
 const OUT_FILE = path.join(__dirname, '..', 'data', 'benchmarks.json');
 
 // ─── helpers ────────────────────────────────────────────────────────────────
 
-async function getJson(url) {
-  const res = await fetch(url, {
-    headers: { 'User-Agent': 'providers-benchmark-fetcher', Accept: 'application/json' },
-  });
-  if (!res.ok) throw new Error(`HTTP ${res.status} from ${url}`);
-  return res.json();
-}
-
 const normName = (s) =>
   (s || '').toLowerCase().replace(/[-_.]/g, ' ').replace(/[^a-z0-9 ]/g, '').replace(/\s+/g, ' ').trim();
 
@@ -241,9 +234,7 @@ async function fetchLiveBench() {
   console.log(`${dates.length} releases (${dates[0]} → ${dates[dates.length - 1]})`);
 
   // Use task→group mapping from the latest categories JSON (stable across releases)
-  const cats = await fetch(`${LB_BASE_URL}/categories_${dates[dates.length - 1]}.json`, {
-    headers: { 'User-Agent': 'providers-benchmark-fetcher' },
-  }).then((r) => r.json());
+  const cats = await getJson(`${LB_BASE_URL}/categories_${dates[dates.length - 1]}.json`);
 
   const taskToGroup = {};
   for (const [cat, tasks] of Object.entries(cats)) {
@@ -263,9 +254,7 @@ async function fetchLiveBench() {
   for (const date of dates) {
     let csv;
     try {
-      csv = await fetch(`${LB_BASE_URL}/table_${date}.csv`, {
-        headers: { 'User-Agent': 'providers-benchmark-fetcher' },
-      }).then((r) => { if (!r.ok) throw new Error(`${r.status}`); return r.text(); });
+      csv = await getText(`${LB_BASE_URL}/table_${date}.csv`);
     } catch (e) {
       console.warn(`\n  ⚠ LiveBench ${date}: ${e.message}`);
       continue;
@@ -393,15 +382,12 @@ async function fetchChatbotArena() {
   // Requesting with "RSC: 1" returns a streaming text/x-component payload that
   // embeds the full leaderboard entries (rank, ELO rating, votes) in the server
   // response — no authentication required.
-  const text = await fetch('https://lmarena.ai/en/leaderboard/text', {
+  const text = await getText('https://lmarena.ai/en/leaderboard/text', {
     headers: {
       'User-Agent': 'Mozilla/5.0',
       'RSC': '1',
       'Accept': 'text/x-component',
     },
-  }).then((r) => {
-    if (!r.ok) throw new Error(`HTTP ${r.status}`);
-    return r.text();
   });
 
   // Each RSC line has the format: <hex_id>:<json_value>
@@ -468,10 +454,7 @@ const AIDER_RAW = 'https://raw.githubusercontent.com/Aider-AI/aider/main/aider/w
 
 async function fetchAider() {
   process.stdout.write('Aider: fetching edit leaderboard... ');
-  const text = await fetch(AIDER_RAW, { headers: { 'User-Agent': 'providers-benchmark-fetcher' } }).then((r) => {
-    if (!r.ok) throw new Error(`HTTP ${r.status}`);
-    return r.text();
-  });
+  const text = await getText(AIDER_RAW);
 
   const rows = yaml.load(text);
 

scripts/fetch-utils.js

@@ -0,0 +1,69 @@
+'use strict';
+
+/**
+ * Robust fetch helper with retries, exponential backoff, and timeout.
+ */
+async function fetchRobust(url, options = {}) {
+  const { retries = 5, backoff = 1000, timeout = 60000, ...fetchOptions } = options;
+  let lastError;
+
+  for (let i = 0; i < retries; i++) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+      const res = await fetch(url, {
+        ...fetchOptions,
+        signal: controller.signal,
+        headers: {
+          'User-Agent': 'providers-benchmark-fetcher',
+          ...fetchOptions.headers,
+        },
+      });
+      clearTimeout(timer);
+
+      if (res.ok) return res;
+
+      // Retry on transient status codes: 429 (Rate Limit) and 5xx (Server Errors)
+      if (res.status === 429 || (res.status >= 500 && res.status < 600)) {
+        lastError = new Error(`HTTP ${res.status} from ${url}`);
+        if (i < retries - 1) {
+          const delay = backoff * Math.pow(2, i) + Math.random() * 1000;
+          process.stdout.write(`\n  ⚠ ${lastError.message}. Retrying in ${Math.round(delay)}ms... (${i + 1}/${retries})\n`);
+          await new Promise((r) => setTimeout(r, delay));
+          continue;
+        }
+      } else {
+        // Don't retry on other 4xx errors (e.g. 404, 401, 403)
+        throw new Error(`HTTP ${res.status} from ${url}`);
+      }
+    } catch (err) {
+      clearTimeout(timer);
+      lastError = err;
+      const isTimeout = err.name === 'AbortError';
+      if (i < retries - 1) {
+        const delay = backoff * Math.pow(2, i) + Math.random() * 1000;
+        const msg = isTimeout ? `Timeout after ${timeout}ms` : err.message;
+        process.stdout.write(`\n  ⚠ Fetch error from ${url}: ${msg}. Retrying in ${Math.round(delay)}ms... (${i + 1}/${retries})\n`);
+        await new Promise((r) => setTimeout(r, delay));
+        continue;
+      }
+    }
+  }
+  throw lastError;
+}
+
+async function getJson(url, options = {}) {
+  const res = await fetchRobust(url, { ...options, headers: { Accept: 'application/json', ...options.headers } });
+  return res.json();
+}
+
+async function getText(url, options = {}) {
+  const res = await fetchRobust(url, options);
+  return res.text();
+}
+
+module.exports = {
+  fetchRobust,
+  getJson,
+  getText,
+};

scripts/providers/black-forest-labs.js

@@ -17,6 +17,7 @@
  */
 
 const URL = 'https://bfl.ai/pricing';
+const { getText } = require('../fetch-utils');
 
 // Extract plain text from Sanity portable text blocks.
 function portableTextToString(blocks) {
@@ -52,7 +53,7 @@ function extractJsonArray(str, marker) {
 }
 
 async function fetchBfl() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -60,9 +61,6 @@ async function fetchBfl() {
     },
   });
 
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
-
   // Decode all RSC payload chunks
   let combined = '';
   const pushRe = /self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)/g;

scripts/providers/groq.js

@@ -14,6 +14,7 @@
  */
 
 const cheerio = require('cheerio');
+const { getText } = require('../fetch-utils');
 
 const URL = 'https://groq.com/pricing';
 
@@ -40,16 +41,13 @@ const cellText = ($, cell) => {
 };
 
 async function fetchGroq() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Language': 'en-US,en;q=0.9',
     },
   });
-
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
   const $ = cheerio.load(html);
 
   const models = [];

scripts/providers/infomaniak.js

@@ -20,6 +20,7 @@
  */
 
 const cheerio = require('cheerio');
+const { getText } = require('../fetch-utils');
 
 const URL = 'https://www.infomaniak.com/en/hosting/ai-services/prices';
 
@@ -44,16 +45,13 @@ const inferType = (name) => {
 };
 
 async function fetchInfomaniak() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Language': 'en-US,en;q=0.9',
     },
   });
-
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
   const $ = cheerio.load(html);
 
   const models = [];

scripts/providers/ionos.js

@@ -18,6 +18,7 @@
  */
 
 const cheerio = require('cheerio');
+const { getText } = require('../fetch-utils');
 
 const URL = 'https://cloud.ionos.com/managed/ai-model-hub';
 
@@ -40,16 +41,13 @@ const splitModels = (text) =>
     .filter(Boolean);
 
 async function fetchIonos() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Language': 'en-US,en;q=0.9',
     },
   });
-
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
   const $ = cheerio.load(html);
 
   const models = [];

scripts/providers/langdock.js

@@ -21,6 +21,7 @@
 
 const cheerio = require('cheerio');
 const { loadEnv } = require('../load-env');
+const { getText } = require('../fetch-utils');
 loadEnv();
 
 const MODELS_URL = 'https://langdock.com/models';
@@ -39,16 +40,13 @@ const getSizeB = (name) => {
 };
 
 async function fetchLangdock() {
-  const response = await fetch(MODELS_URL, {
+  const html = await getText(MODELS_URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       'Accept-Language': 'en-US,en;q=0.9',
     },
   });
-
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${MODELS_URL}`);
-  const html = await response.text();
   const $ = cheerio.load(html);
 
   const models = [];

scripts/providers/mistral.js

@@ -10,6 +10,7 @@
  */
 
 const URL = 'https://mistral.ai/pricing';
+const { getText } = require('../fetch-utils');
 
 const stripHtml = (html) => (html || '').replace(/<[^>]+>/g, '').trim();
 
@@ -63,7 +64,7 @@ function extractApisArray(payload) {
 }
 
 async function fetchMistral() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -71,9 +72,6 @@ async function fetchMistral() {
     },
   });
 
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
-
   // The page uses Next.js App Router RSC streaming. Pricing data is in a
   // self.__next_f.push([1, "ENCODED_STRING"]) script tag. Inside the raw HTML,
   // inner quotes are escaped as \" so we search for the literal \\"apis\\":[{

scripts/providers/nebius.js

@@ -13,6 +13,8 @@
  *   ['Model','Input']                    – image gen / embeddings; single rows
  */
 
+const { getText } = require('../fetch-utils');
+
 const URL = 'https://nebius.com/token-factory/prices';
 
 const parseUsd = (text) => {
@@ -122,7 +124,7 @@ function modelsFromTable({ rows }) {
 }
 
 async function fetchNebius() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -130,8 +132,6 @@ async function fetchNebius() {
     },
   });
 
-  if (!response.ok) throw new Error(`HTTP ${response.status} from ${URL}`);
-  const html = await response.text();
   if (html.includes('cf-browser-verification') || html.includes('Just a moment')) {
     throw new Error('Blocked by Cloudflare');
   }

scripts/providers/openrouter.js

@@ -9,6 +9,7 @@
 
 const { loadEnv } = require('../load-env');
 loadEnv();
+const { getJson } = require('../fetch-utils');
 
 const API_URL = 'https://openrouter.ai/api/v1/models';
 
@@ -61,13 +62,7 @@ async function fetchOpenRouter() {
   };
   if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
 
-  const response = await fetch(API_URL, { headers });
-
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status} from OpenRouter API`);
-  }
-
-  const data = await response.json();
+  const data = await getJson(API_URL, { headers });
   const models = [];
 
   for (const model of data.data || []) {

scripts/providers/requesty.js

@@ -2,6 +2,7 @@
 
 const { loadEnv } = require('../load-env');
 loadEnv();
+const { getJson } = require('../fetch-utils');
 
 const API_URL = 'https://router.requesty.ai/v1/models';
 
@@ -23,18 +24,13 @@ async function fetchRequesty() {
     return [];
   }
 
-  const response = await fetch(API_URL, {
+  const data = await getJson(API_URL, {
     headers: {
       Authorization: `Bearer ${apiKey}`,
       Accept: 'application/json',
     },
   });
 
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status} from Requesty API`);
-  }
-
-  const data = await response.json();
   const models = [];
 
   for (const model of data.data || []) {

scripts/providers/scaleway.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const cheerio = require('cheerio');
+const { getText } = require('../fetch-utils');
 
 const URL = 'https://www.scaleway.com/en/pricing/model-as-a-service/';
 
@@ -28,7 +29,7 @@ const getSizeB = (name) => {
 };
 
 async function fetchScaleway() {
-  const response = await fetch(URL, {
+  const html = await getText(URL, {
     headers: {
       'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -37,12 +38,6 @@ async function fetchScaleway() {
     },
   });
 
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status} fetching ${URL}`);
-  }
-
-  const html = await response.text();
-
   // Quick sanity check for Cloudflare block
   if (html.includes('cf-browser-verification') || html.includes('Just a moment')) {
     throw new Error('Blocked by Cloudflare – try adding a delay or using a browser-based fetcher');