Review and analyse this whole Scraperlib library codes after this list all of the best of the best possible js client-side only library you can find or can think of to make this Scraperlib library more powerful, fast, reliable, bug free, use full, effective, efficient, bottleneck free, error free, memory leaks free, resources consumption free, better than before.

Preferable lightweight libraries.

Think really hard before coming to the conclusion.

/*!
 * ScraperLib.js v6.7.0
 * A fully stable, practical, and robust client-side web scraping library ready for production use
 * License: MIT
 */

(function (global) {
  'use strict';

  /**
   * Utility functions for validation and processing
   */
  const Utils = {
    // Validates if a string is a valid URL
    isValidURL(url) {
      if (typeof url !== 'string' || url.length > 2048) return false;
      try {
        new URL(url);
        return true;
      } catch {
        return false;
      }
    },

    // Sanitizes a string to prevent XSS
    sanitize(str) {
      if (typeof str !== 'string') return '';
      const temp = document.createElement('div');
      temp.textContent = str;
      return temp.innerHTML;
    },

    // Initializes the cache if available
    async initCache() {
      if ('caches' in global) {
        try {
          return await caches.open('scraper-lib-cache');
        } catch (e) {
          console.error('Cache initialization failed:', e);
          return null;
        }
      }
      return null;
    },

    // Generates a cache key based on URL and options
    async generateCacheKey(url, options = {}) {
      const cacheOptions = {
        useProxy: options.useProxy,
        proxyUrl: options.useProxy ? options.proxyUrl : undefined,
        selectorsHash: await Utils.hashSelectors(options.selectors),
        userAgent: options.userAgent,
        acceptLanguage: options.acceptLanguage,
        cacheTime: options.cacheTime,
      };
      // Exclude functions from JSON.stringify
      const cacheKeyString = `${url}-${JSON.stringify(cacheOptions, (key, value) => {
        if (typeof value === 'function') {
          return undefined;
        }
        return value;
      })}`;
      const hash = await Utils.hashString(cacheKeyString);
      return hash;
    },

    // Generates a hash for selectors to include in cache key
    async hashSelectors(selectors) {
      if (!selectors) return '';
      try {
        const selectorsString = JSON.stringify(selectors, (key, value) => {
          if (typeof value === 'function') {
            return undefined; // Exclude functions from JSON.stringify
          }
          return value;
        });
        return await Utils.hashString(selectorsString);
      } catch (e) {
        console.error('Error hashing selectors:', e);
        return '';
      }
    },

    // Generates a cryptographic hash from a string
    async hashString(str) {
      if (typeof crypto !== 'undefined' && crypto.subtle) {
        const encoder = new TextEncoder();
        const data = encoder.encode(str);
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        const hashArray = Array.from(new Uint8Array(hashBuffer));
        const hashHex = hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
        return hashHex;
      } else {
        // Fallback to a simple hash function
        let hash = 0,
          i,
          chr;
        if (str.length === 0) return hash.toString();
        for (i = 0; i < str.length; i++) {
          chr = str.charCodeAt(i);
          hash = (hash << 5) - hash + chr;
          hash |= 0; // Convert to 32bit integer
        }
        return hash.toString();
      }
    },

    // Validates and normalizes CSS selectors
    normalizeSelector(selector) {
      if (typeof selector !== 'string') throw new Error('Selector must be a string');
      try {
        document.createDocumentFragment().querySelector(selector);
        return selector;
      } catch {
        throw new Error(`Invalid CSS selector: "${selector}"`);
      }
    },

    // Deep merge of objects
    deepMerge(target, ...sources) {
      target = target || {};
      sources.forEach(src => {
        if (src && typeof src === 'object') {
          Object.keys(src).forEach(key => {
            const value = src[key];
            if (
              value &&
              typeof value === 'object' &&
              !Array.isArray(value) &&
              !(value instanceof RegExp)
            ) {
              target[key] = Utils.deepMerge(target[key], value);
            } else {
              target[key] = value;
            }
          });
        }
      });
      return target;
    },

    // Extracts text from an element, optionally preserving formatting
    extractText(element, options = {}) {
      if (!element) return '';
      const clone = element.cloneNode(true);
      // Remove scripts and styles
      clone.querySelectorAll('script, style, noscript').forEach(el => el.remove());
      if (options.preserveFormatting) {
        // Replace <br> with newline characters
        clone.querySelectorAll('br').forEach(br => {
          const textNode = document.createTextNode('\n');
          br.parentNode.replaceChild(textNode, br);
        });
        // Add newline after </p> elements
        clone.querySelectorAll('p').forEach(p => {
          const textNode = document.createTextNode('\n');
          p.appendChild(textNode);
        });
      }
      return clone.textContent.trim();
    },

    // Removes scripts and styles from HTML string
    removeScriptsAndStyles(html) {
      try {
        const doc = new DOMParser().parseFromString(html, 'text/html');
        doc.querySelectorAll('script, style, noscript').forEach(el => el.remove());
        return doc.documentElement.outerHTML;
      } catch (e) {
        console.error('Error parsing HTML for script/style removal:', e);
        return html
          .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '')
          .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '')
          .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, '');
      }
    },

    // Chunks an array into smaller arrays of specified size
    chunkArray(array, size) {
      if (!Array.isArray(array)) throw new Error('Input must be an array');
      if (!Number.isInteger(size) || size <= 0) throw new Error('Chunk size must be a positive integer');
      const result = [];
      for (let i = 0; i < array.length; i += size) {
        result.push(array.slice(i, i + size));
      }
      return result;
    },

    // Parses a string to JSON safely
    safeParseJSON(str) {
      try {
        return JSON.parse(str);
      } catch {
        return null;
      }
    },

    // Converts a NodeList to an Array
    nodeListToArray(nodeList) {
      return Array.from(nodeList);
    },

    // Function to delay execution
    delay(ms) {
      return new Promise(resolve => setTimeout(resolve, ms));
    },

    // Check for anti-scraping mechanisms (e.g., CAPTCHA)
    detectAntiScraping(html) {
      // Simple keyword detection for common anti-scraping messages
      const antiScrapingKeywords = [
        'captcha',
        'verify you are human',
        'access denied',
        'bot detection',
        'unusual traffic',
        'request blocked',
        'please enable cookies',
      ];
      const lowerHtml = html.toLowerCase();
      return antiScrapingKeywords.some(keyword => lowerHtml.includes(keyword));
    },

    // Built-in filters for common operations
    builtInFilters: {
      trim: value => (typeof value === 'string' ? value.trim() : value),
      toUpperCase: value => (typeof value === 'string' ? value.toUpperCase() : value),
      toLowerCase: value => (typeof value === 'string' ? value.toLowerCase() : value),
      parseInt: value => parseInt(value, 10),
      parseFloat: value => parseFloat(value),
      // Additional built-in filters
      replace: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 2) return value;
        const [pattern, replacement] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern, 'g');
        } else {
          return value;
        }
        return value.replace(regex, replacement);
      },
      substring: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.substring(args[0], args[1]);
      },
      split: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args)) return value;
        return value.split(args[0]);
      },
      join: (value, args) => {
        if (!Array.isArray(value)) return value;
        return value.join(args[0]);
      },
      regexMatch: (value, args) => {
        if (typeof value !== 'string' || !Array.isArray(args) || args.length < 1) return value;
        const [pattern, group = 0] = args;
        let regex;
        if (pattern instanceof RegExp) {
          regex = pattern;
        } else if (typeof pattern === 'string') {
          regex = new RegExp(pattern);
        } else {
          return value;
        }
        const match = value.match(regex);
        return match ? match[group] : null;
      },
      // Additional filters can be added here
    },
  };

  /**
   * Handles storing scraped data
   */
  class StorageHandler {
    constructor() {
      this.dbPromise = null;
      this.storeName = 'scrapedData';
      if ('indexedDB' in global) {
        this.storageType = 'indexeddb';
        this.dbPromise = this.initIndexedDB();
      } else if ('localStorage' in global && this._isLocalStorageAccessible()) {
        this.storageType = 'localstorage';
      } else {
        this.storageType = 'memory';
        this.memoryStorage = new Map();
      }
    }

    _isLocalStorageAccessible() {
      try {
        const testKey = '__test__';
        localStorage.setItem(testKey, testKey);
        localStorage.removeItem(testKey);
        return true;
      } catch {
        console.warn('localStorage is not accessible. Using in-memory storage.');
        return false;
      }
    }

    async initIndexedDB(dbName = 'ScraperLibDB') {
      return new Promise((resolve, reject) => {
        const request = indexedDB.open(dbName, 1);

        request.onerror = () => reject(new Error('IndexedDB initialization failed'));

        request.onupgradeneeded = event => {
          const db = event.target.result;
          if (!db.objectStoreNames.contains(this.storeName)) {
            const store = db.createObjectStore(this.storeName, { keyPath: 'id', autoIncrement: true });
            store.createIndex('url', 'url', { unique: false });
            store.createIndex('timestamp', 'timestamp', { unique: false });
          }
        };

        request.onsuccess = event => {
          const db = event.target.result;
          db.onerror = event => {
            console.error('Database error:', event.target.error);
          };
          resolve(db);
        };
      });
    }

    async store(url, data, format, options = {}) {
      const timestamp = Date.now();
      const storeData = { url, data, timestamp };

      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readwrite');
              const store = transaction.objectStore(this.storeName);
              const request = store.add(storeData);

              request.onsuccess = () => resolve(true);
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          try {
            const key = `scraper-${url}-${timestamp}`;
            localStorage.setItem(key, JSON.stringify(storeData));
            return true;
          } catch (e) {
            if (e.name === 'QuotaExceededError' || e.name === 'NS_ERROR_DOM_QUOTA_REACHED') {
              this._cleanupLocalStorage();
              const key = `scraper-${url}-${timestamp}`;
              localStorage.setItem(key, JSON.stringify(storeData));
              return true;
            }
            throw e;
          }

        case 'memory':
          this.memoryStorage.set(`${url}-${timestamp}`, storeData);
          return true;

        case 'custom':
          if (options.customStore && typeof options.customStore.save === 'function') {
            await options.customStore.save(url, data);
            return true;
          }
          throw new Error('Invalid custom storage handler');

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }

    _cleanupLocalStorage() {
      const keys = [];
      for (let i = 0; i < localStorage.length; i++) {
        const key = localStorage.key(i);
        if (key && key.startsWith('scraper-')) {
          keys.push(key);
        }
      }
      if (keys.length > 100) {
        keys.sort((a, b) => {
          try {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return itemA.timestamp - itemB.timestamp;
          } catch (e) {
            localStorage.removeItem(a);
            localStorage.removeItem(b);
            return 0;
          }
        });
        for (let i = 0; i < keys.length - 100; i++) {
          localStorage.removeItem(keys[i]);
        }
      }
    }

    async retrieve(url, format) {
      switch (format) {
        case 'indexeddb':
          if (this.dbPromise) {
            const db = await this.dbPromise;
            return new Promise((resolve, reject) => {
              const transaction = db.transaction(this.storeName, 'readonly');
              const store = transaction.objectStore(this.storeName);
              const index = store.index('url');
              const request = index.getAll(url);

              request.onsuccess = () => {
                const results = request.result;
                if (results && results.length > 0) {
                  results.sort((a, b) => b.timestamp - a.timestamp);
                  resolve(results[0].data);
                } else {
                  resolve(null);
                }
              };
              request.onerror = () => reject(request.error);
            });
          }
          throw new Error('IndexedDB is not supported or initialized');

        case 'localstorage':
          const keys = [];
          for (let i = 0; i < localStorage.length; i++) {
            const key = localStorage.key(i);
            if (key && key.startsWith(`scraper-${url}-`)) {
              keys.push(key);
            }
          }
          keys.sort((a, b) => {
            const itemA = Utils.safeParseJSON(localStorage.getItem(a));
            const itemB = Utils.safeParseJSON(localStorage.getItem(b));
            return (itemB?.timestamp || 0) - (itemA?.timestamp || 0);
          });
          if (keys.length > 0) {
            const item = Utils.safeParseJSON(localStorage.getItem(keys[0]));
            return item ? item.data : null;
          }
          return null;

        case 'memory':
          const memoryItems = Array.from(this.memoryStorage.entries())
            .filter(([key]) => key.startsWith(`${url}-`))
            .sort((a, b) => b[1].timestamp - a[1].timestamp);
          if (memoryItems.length > 0) {
            return memoryItems[0][1].data;
          }
          return null;

        default:
          throw new Error(`Unsupported storage format: ${format}`);
      }
    }
  }

  /**
   * Handles rate limiting to respect website policies
   */
  class RateLimiter {
    constructor(requestsPerSecond = 2) {
      this.requestsPerSecond = Math.max(1, requestsPerSecond);
      this.interval = 1000 / this.requestsPerSecond;
      this.queue = [];
      this.lastRequestTime = 0;
      this.isProcessing = false;
    }

    async acquireToken() {
      return new Promise(resolve => {
        this.queue.push(resolve);
        this._processQueue();
      });
    }

    _processQueue() {
      if (this.isProcessing || this.queue.length === 0) return;
      this.isProcessing = true;

      const now = Date.now();
      const elapsed = now - this.lastRequestTime;
      const delay = Math.max(this.interval - elapsed, 0);

      setTimeout(() => {
        this.lastRequestTime = Date.now();
        const resolve = this.queue.shift();
        if (resolve) resolve();
        this.isProcessing = false;
        if (this.queue.length > 0) {
          this._processQueue();
        }
      }, delay);
    }
  }

  /**
   * Main ScraperLib class
   */
  class ScraperLib {
    constructor(options = {}) {
      this.defaultSettings = {
        rateLimitRequests: 2,
        maxConcurrent: 5,
        timeout: 30000,
        retries: 3,
        retryDelay: 1000,
        maxRetryDelay: 30000,
        cacheTime: 3600000,
        outputFormat: 'json',
        useCache: false,
        headers: {},
        preserveFormatting: false,
        validateSelectors: true,
        maxResponseSize: 10 * 1024 * 1024,
        useProxy: false,
        proxyUrl: '',
        sinceModified: false,
        filters: {}, // Global filters
        userAgent: '', // Cannot be set in fetch in browsers
        acceptLanguage: '', // Cannot be set in fetch in browsers
        concurrencyStrategy: 'auto', // 'auto', 'parallel', 'sequential'
        logger: null, // Custom logger function
        customFetch: null, // Custom fetch function
        politenessDelay: 0, // Delay between requests to the same domain
        maxRetriesOnAntiScraping: 2, // Max retries if anti-scraping measures detected
        builtInFilters: Utils.builtInFilters, // Reference to built-in filters
        enableLogging: true, // Enable or disable logging
      };

      // Forbidden headers in browsers
      this.forbiddenHeaders = new Set([
        'accept-charset',
        'accept-encoding',
        'access-control-request-headers',
        'access-control-request-method',
        'connection',
        'content-length',
        'cookie',
        'cookie2',
        'date',
        'dnt',
        'expect',
        'host',
        'keep-alive',
        'origin',
        'referer',
        'te',
        'trailer',
        'transfer-encoding',
        'upgrade',
        'via',
        'proxy-',
        'sec-',
        'user-agent',
        'accept-language',
      ]);

      // Merge and validate default settings with user options
      this.settings = Utils.deepMerge({}, this.defaultSettings, options);
      this._validateSettings(this.settings);

      // Initialize per-URL settings
      this.urlSettings = new Map();

      this.storage = new StorageHandler();
      this.rateLimiter = new RateLimiter(this.settings.rateLimitRequests);
      this.cache = null;
      this.errorHandlers = new Map();

      // Initialize cache
      if (this.settings.useCache) {
        this.cachePromise = Utils.initCache().then(cache => {
          this.cache = cache;
        });
      } else {
        this.cachePromise = Promise.resolve();
      }

      // Initialize logger
      this.logger = this.settings.logger || console;

      // Hostname to last request time mapping for politeness
      this.hostLastRequestTime = new Map();
    }

    _validateSettings(settings) {
      const positiveNumberProps = [
        'rateLimitRequests',
        'maxConcurrent',
        'retries',
        'retryDelay',
        'maxRetryDelay',
        'cacheTime',
        'timeout',
        'maxResponseSize',
        'politenessDelay',
        'maxRetriesOnAntiScraping',
      ];
      positiveNumberProps.forEach(prop => {
        if (typeof settings[prop] !== 'number' || settings[prop] < 0) {
          throw new Error(`Invalid setting "${prop}": must be a positive number`);
        }
      });

      const allowedStrategies = ['auto', 'parallel', 'sequential'];
      if (!allowedStrategies.includes(settings.concurrencyStrategy)) {
        throw new Error(
          `Invalid concurrency strategy: "${settings.concurrencyStrategy}". Allowed values are ${allowedStrategies.join(
            ', '
          )}`
        );
      }
    }

    /**
     * Main scraping method
     */
    async scrape(configs, globalOptions = {}) {
      await this.cachePromise;

      if (typeof configs !== 'object' || configs === null || Array.isArray(configs)) {
        throw new Error('Configs must be an object with URL keys and configuration values');
      }

      // Merge global options into settings
      const globalSettings = Utils.deepMerge({}, this.settings, globalOptions);
      this._validateSettings(globalSettings);

      const results = {};
      const urls = Object.keys(configs);
      if (urls.length === 0) {
        return results;
      }

      await this._log('Starting scraping process', { urls });

      // Prepare per-domain request queues to respect politeness
      switch (globalSettings.concurrencyStrategy) {
        case 'parallel':
          await Promise.all(
            urls.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
          );
          break;

        case 'sequential':
          for (const url of urls) {
            await this._scrapeUrlWrapper(url, configs[url], globalSettings, results);
          }
          break;

        case 'auto':
        default:
          const chunks = Utils.chunkArray(urls, globalSettings.maxConcurrent);
          for (const chunk of chunks) {
            await Promise.all(
              chunk.map(url => this._scrapeUrlWrapper(url, configs[url], globalSettings, results))
            );
          }
          break;
      }

      await this._log('Completed scraping process', { results });

      return results;
    }

    /**
     * Wrapper for processing a URL, handles logging and error handling
     */
    async _scrapeUrlWrapper(url, config, globalSettings, results) {
      try {
        const data = await this._processUrl(url, config, globalSettings);
        results[url] = data;
      } catch (error) {
        const handler = this.errorHandlers.get(url) || this.errorHandlers.get('default');
        if (handler) {
          try {
            const data = await handler(error, url);
            results[url] = data;
          } catch (e) {
            results[url] = { error: e.message };
          }
        } else {
          results[url] = { error: error.message };
        }
        await this._log(`Error scraping ${url}: ${error.message}`, { error });
      }
    }

    /**
     * Process a single URL with retries and error handling
     */
    async _processUrl(url, config, globalSettings) {
      if (!Utils.isValidURL(url)) {
        throw new Error(`Invalid URL: ${url}`);
      }

      // Merge global settings with per-URL settings using deep merge
      const settings = Utils.deepMerge({}, globalSettings, config.options);
      this._validateSettings(settings);
      this.urlSettings.set(url, settings);

      if (settings.validateSelectors) {
        this._validateSelectors(config.selectors);
      }

      // Wait for cache initialization if needed
      if (settings.useCache) {
        await this.cachePromise;
        if (this.cache) {
          const cachedData = await this._getFromCache(url, settings);
          if (cachedData) {
            await this._log(`Using cached data for ${url}`);
            return cachedData;
          }
        }
      }

      let attempts = 0;
      let antiScrapingAttempts = 0;
      while (attempts < settings.retries) {
        attempts += 1;
        await this._respectPoliteness(url, settings);

        await this.rateLimiter.acquireToken();
        const options = this._prepareFetchOptions(config, settings);
        try {
          const response = await this._fetchWithTimeout(url, options, settings);

          if (!response.ok) {
            if (
              response.status === 304 &&
              settings.useCache &&
              this.cache &&
              settings.sinceModified
            ) {
              const cachedData = await this._getFromCache(url, settings);
              if (cachedData) {
                return cachedData;
              }
            }
            throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
          }

          const contentType = response.headers.get('Content-Type') || '';
          if (!contentType.includes('text/html')) {
            throw new Error(`Invalid Content-Type: ${contentType}`);
          }

          const data = await this._extractDataFromResponse(response, config.selectors, settings);

          if (settings.outputFormat !== 'json') {
            await this.storage.store(url, data, settings.outputFormat, { customStore: settings.customStore });
          }

          if (settings.useCache && this.cache) {
            await this._storeInCache(url, data, settings);
          }

          return data;
        } catch (error) {
          if (
            Utils.detectAntiScraping(error.message) &&
            antiScrapingAttempts < settings.maxRetriesOnAntiScraping
          ) {
            antiScrapingAttempts += 1;
            await this._log(
              `Detected anti-scraping measures on ${url}. Attempt ${antiScrapingAttempts}`,
              { error }
            );
            // Delay before retrying
            await Utils.delay(settings.retryDelay * antiScrapingAttempts);
            continue;
          }

          if (attempts >= settings.retries) {
            throw error;
          }
          // Exponential backoff with jitter
          const delay = Math.min(
            settings.retryDelay * Math.pow(2, attempts - 1) * (0.5 + Math.random() * 0.5),
            settings.maxRetryDelay
          );
          await this._log(`Retrying ${url} after delay of ${delay}ms`, { attempt: attempts, error });
          await Utils.delay(delay);
        }
      }
    }

    /**
     * Respects politeness policies by delaying requests to the same domain
     */
    async _respectPoliteness(url, settings) {
      const urlObj = new URL(url);
      const hostname = urlObj.hostname;
      const now = Date.now();
      const lastRequestTime = this.hostLastRequestTime.get(hostname) || 0;
      const elapsed = now - lastRequestTime;

      if (elapsed < settings.politenessDelay) {
        const delay = settings.politenessDelay - elapsed;
        await this._log(`Delaying request to ${url} by ${delay}ms for politeness`);
        await Utils.delay(delay);
      }

      this.hostLastRequestTime.set(hostname, Date.now());
    }

    /**
     * Validates the CSS selectors
     */
    _validateSelectors(selectors) {
      if (typeof selectors !== 'object' || selectors === null) {
        throw new Error('Selectors must be an object');
      }
      for (const key of Object.keys(selectors)) {
        const sel = selectors[key];
        if (typeof sel === 'string') {
          Utils.normalizeSelector(sel);
        } else if (sel && typeof sel.selector === 'string') {
          Utils.normalizeSelector(sel.selector);
        } else if (sel && typeof sel.customSelector === 'function') {
          // Custom selector function, assume valid
        } else if (sel && typeof sel.xpath === 'string') {
          // Could validate XPath here if needed
        } else {
          throw new Error(`Invalid selector configuration for key "${key}"`);
        }
      }
    }

    /**
     * Prepares fetch options including headers
     */
    _prepareFetchOptions(config, settings) {
      const headersInit = Utils.deepMerge({}, settings.headers, config.headers);

      // Remove forbidden headers
      Object.keys(headersInit).forEach(header => {
        const headerLower = header.toLowerCase();
        if (
          this.forbiddenHeaders.has(headerLower) ||
          Array.from(this.forbiddenHeaders).some(
            fh => fh.endsWith('-') && headerLower.startsWith(fh)
          )
        ) {
          delete headersInit[header];
        }
      });

      const headers = new Headers(headersInit);

      if (settings.sinceModified) {
        const lastModified = new Date(Date.now() - settings.cacheTime).toUTCString();
        headers.set('If-Modified-Since', lastModified);
      }

      const options = {
        method: 'GET',
        headers: headers,
        mode: 'cors',
        credentials: 'omit',
        redirect: 'follow',
      };

      return options;
    }

    /**
     * Performs a fetch request with a timeout, using proxy if configured
     */
    async _fetchWithTimeout(url, options, settings) {
      const controller = new AbortController();
      options.signal = controller.signal;

      const timeoutId = setTimeout(() => {
        controller.abort();
      }, settings.timeout);

      try {
        let fetchUrl = url;
        if (settings.useProxy) {
          if (!settings.proxyUrl) {
            throw new Error('Proxy URL is not set');
          }
          // Correctly construct the fetch URL with the proxy
          if (settings.proxyUrl.includes('${url}')) {
            fetchUrl = settings.proxyUrl.replace('${url}', encodeURIComponent(url));
          } else {
            // Append the encoded URL appropriately
            if (settings.proxyUrl.endsWith('=') || settings.proxyUrl.endsWith('?')) {
              fetchUrl = settings.proxyUrl + encodeURIComponent(url);
            } else {
              // Append '?url=' + encodeURIComponent(url)
              const separator = settings.proxyUrl.includes('?') ? '&' : '?';
              fetchUrl = settings.proxyUrl + separator + 'url=' + encodeURIComponent(url);
            }
          }
        }

        const fetchFunction = settings.customFetch || fetch;

        const response = await fetchFunction(fetchUrl, options);
        clearTimeout(timeoutId);

        // Check if response size exceeds maxResponseSize
        const contentLength = response.headers.get('Content-Length');
        if (contentLength && parseInt(contentLength, 10) > settings.maxResponseSize) {
          throw new Error('Response size exceeds maximum allowed');
        }

        return response;
      } catch (error) {
        clearTimeout(timeoutId);
        if (error.name === 'AbortError') {
          throw new Error('Request timed out');
        }
        throw error;
      }
    }

    /**
     * Enhanced data extraction with advanced filters
     */
    async _extractDataFromResponse(response, selectors, settings) {
      try {
        const decoder = new TextDecoder('utf-8');
        const reader = response.body.getReader();
        let receivedLength = 0;
        let chunks = [];
        while (true) {
          const { done, value } = await reader.read();
          if (done) break;
          receivedLength += value.length;
          if (receivedLength > settings.maxResponseSize) {
            throw new Error('Response size exceeds maximum allowed');
          }
          chunks.push(value);
        }
        const htmlText = decoder.decode(ScraperLib.concatChunks(chunks));

        // Check for anti-scraping mechanisms
        if (Utils.detectAntiScraping(htmlText)) {
          throw new Error('Detected anti-scraping measures in the response');
        }

        // Remove scripts and styles to prevent XSS and reduce parsing overhead
        const sanitizedHtml = Utils.removeScriptsAndStyles(htmlText);

        const parser = new DOMParser();
        const doc = parser.parseFromString(sanitizedHtml, 'text/html');

        const data = {};

        for (const key of Object.keys(selectors)) {
          const selConfig = selectors[key];

          let elements;
          if (typeof selConfig === 'string') {
            elements = doc.querySelectorAll(selConfig);
          } else if (selConfig && typeof selConfig.selector === 'string') {
            elements = doc.querySelectorAll(selConfig.selector);
          } else if (selConfig && typeof selConfig.customSelector === 'function') {
            elements = selConfig.customSelector(doc);
            if (!Array.isArray(elements)) {
              throw new Error(`customSelector for key "${key}" must return an array of elements`);
            }
          } else if (selConfig && typeof selConfig.xpath === 'string') {
            elements = this._selectXPath(doc, selConfig.xpath);
          } else {
            throw new Error(`Invalid selector configuration for key "${key}"`);
          }

          if (!elements || elements.length === 0) {
            data[key] = selConfig.single ? null : [];
            continue;
          }

          data[key] = await Promise.all(
            Array.from(elements).map(async element => {
              let value;
              if (selConfig.attr) {
                value = element.getAttribute(selConfig.attr) || '';
              } else if (selConfig.html) {
                value = element.innerHTML;
              } else {
                value = Utils.extractText(element, { preserveFormatting: settings.preserveFormatting });
              }

              // Apply per-selector filters (support chaining)
              if (selConfig.filters) {
                value = await this._applyFilters(value, selConfig.filters, element, {
                  key,
                  settings,
                });
              }

              // Apply global filters if no per-selector filters
              else if (settings.filters && settings.filters[key]) {
                value = await this._applyFilters(value, settings.filters[key], element, {
                  key,
                  settings,
                });
              }

              return value;
            })
          );

          data[key] = data[key].filter(val => val !== undefined && val !== null);

          if (selConfig.single) {
            data[key] = data[key][0] || null;
          }
        }

        return data;
      } catch (e) {
        throw new Error(`Failed to extract data: ${e.message}`);
      }
    }

    /**
     * Apply filters to a value, supporting chaining and built-in filters
     */
    async _applyFilters(value, filters, element, context) {
      if (!filters) return value;

      const filtersArray = Array.isArray(filters) ? filters : [filters];

      for (let filter of filtersArray) {
        if (typeof filter === 'string') {
          // Built-in filter by name
          const builtInFilter = this._parseFilterString(filter);
          if (typeof builtInFilter === 'function') {
            value = builtInFilter(value, element, context);
          } else {
            console.warn(`Built-in filter "${filter}" not found`);
          }
        } else if (typeof filter === 'function') {
          try {
            const filterResult = filter(value, element, context);
            value = filterResult instanceof Promise ? await filterResult : filterResult;
          } catch (e) {
            console.error(`Error in filter for key "${context.key}":`, e);
            value = null;
            break;
          }
        } else if (filter instanceof RegExp) {
          // Apply regular expression
          const match = value.match(filter);
          value = match ? match[0] : null;
        } else {
          console.warn(`Invalid filter type for key "${context.key}"`);
        }

        // If value becomes null or undefined, stop processing filters
        if (value == null) {
          break;
        }
      }

      return value;
    }

    /**
     * Parse filter string to support filter arguments safely
     */
    _parseFilterString(filterStr) {
      // Example filter string: 'replace(/foo/g, "bar")'
      const filterPattern = /^(\w+)\((.*)\)$/;
      const match = filterStr.match(filterPattern);
      if (match) {
        const filterName = match[1];
        const argsStr = match[2];
        const args = this._parseFilterArgs(argsStr);

        const filterFunc = this.settings.builtInFilters[filterName];
        if (typeof filterFunc === 'function') {
          return (value, element, context) => filterFunc(value, args, element, context);
        }
      } else {
        const filterFunc = this.settings.builtInFilters[filterStr];
        if (typeof filterFunc === 'function') {
          return filterFunc;
        }
      }
      return null;
    }

    /**
     * Parse filter arguments from string safely
     */
    _parseFilterArgs(argsStr) {
      try {
        const args = [];
        let currentArg = '';
        let inQuotes = false;
        let quoteChar = '';
        let escapeNext = false;
        let bracketDepth = 0;

        for (let i = 0; i < argsStr.length; i++) {
          const char = argsStr.charAt(i);

          if (escapeNext) {
            currentArg += char;
            escapeNext = false;
            continue;
          }

          if (char === '\\') {
            escapeNext = true;
            continue;
          }

          if (inQuotes) {
            if (char === quoteChar) {
              inQuotes = false;
            }
            currentArg += char;
            continue;
          }

          if (char === '"' || char === "'") {
            inQuotes = true;
            quoteChar = char;
            currentArg += char;
            continue;
          }

          if (char === '(') {
            bracketDepth++;
          } else if (char === ')') {
            bracketDepth--;
          }

          if (char === ',' && !inQuotes && bracketDepth === 0) {
            args.push(this._parseArgument(currentArg.trim()));
            currentArg = '';
            continue;
          }

          currentArg += char;
        }

        if (currentArg) {
          args.push(this._parseArgument(currentArg.trim()));
        }

        return args;
      } catch (e) {
        console.error('Failed to parse filter arguments:', e);
        return [];
      }
    }

    _parseArgument(arg) {
      if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) {
        // String argument
        return arg.slice(1, -1).replace(/\\'/g, "'").replace(/\\"/g, '"');
      }

      // Try to parse as number
      const num = Number(arg);
      if (!isNaN(num)) {
        return num;
      }

      // For regex arguments, e.g., /pattern/flags
      if (arg.startsWith('/') && arg.lastIndexOf('/') > 0) {
        const lastSlash = arg.lastIndexOf('/');
        const pattern = arg.slice(1, lastSlash);
        const flags = arg.slice(lastSlash + 1);
        try {
          return new RegExp(pattern, flags);
        } catch (e) {
          console.error('Invalid regular expression:', e);
          return null;
        }
      }

      // Boolean arguments
      if (arg === 'true') return true;
      if (arg === 'false') return false;

      // Otherwise, return the arg as string
      return arg;
    }

    /**
     * Select elements using XPath
     */
    _selectXPath(doc, xpathExpression) {
      const iterator = doc.evaluate(xpathExpression, doc, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
      const nodes = [];
      for (let i = 0; i < iterator.snapshotLength; i++) {
        nodes.push(iterator.snapshotItem(i));
      }
      return nodes;
    }

    /**
     * Concatenates array of Uint8Arrays into one Uint8Array
     */
    static concatChunks(chunks) {
      let totalLength = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
      let result = new Uint8Array(totalLength);
      let position = 0;
      for (let chunk of chunks) {
        result.set(chunk, position);
        position += chunk.length;
      }
      return result;
    }

    /**
     * Gets data from the cache if available and not expired
     */
    async _getFromCache(url, settings) {
      if (!this.cache) return null;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = await this.cache.match(cacheKey);

      if (cachedResponse) {
        try {
          const cachedData = await cachedResponse.json();
          const age = Date.now() - cachedData.timestamp;
          if (age < settings.cacheTime) {
            return cachedData.data;
          }
        } catch (e) {
          console.warn('Error reading from cache:', e);
        }
      }

      return null;
    }

    /**
     * Stores data in the cache
     */
    async _storeInCache(url, data, settings) {
      if (!this.cache) return;
      const cacheKey = await Utils.generateCacheKey(url, settings);
      const cachedResponse = new Response(
        JSON.stringify({ data, timestamp: Date.now() }),
        {
          headers: { 'Content-Type': 'application/json' },
        }
      );
      try {
        await this.cache.put(cacheKey, cachedResponse);
      } catch (e) {
        console.error('Failed to store data in cache:', e);
      }
    }

    /**
     * Registers a custom error handler
     */
    onError(urlOrDefault, handler) {
      if (typeof handler !== 'function') {
        throw new Error('Error handler must be a function');
      }
      this.errorHandlers.set(urlOrDefault, handler);
    }

    /**
     * Logs messages using the provided logger
     */
    async _log(message, data = {}) {
      if (!this.settings.enableLogging) return;
      if (typeof this.logger === 'function') {
        this.logger(message, data);
      } else if (this.logger && typeof this.logger.log === 'function') {
        this.logger.log(message, data);
      }
    }
  }

  // Attach ScraperLib to the global object
  global.ScraperLib = ScraperLib;
})(typeof window !== 'undefined' ? window : this);